├── bitdata ├── __init__.py ├── core │ ├── __init__.py │ └── config.py ├── provider │ ├── __init__.py │ ├── quiknode.py │ ├── blockstream.py │ ├── bitcoin_rpc.py │ └── mempool.py ├── notifiers │ ├── discord.py │ └── telegram.py └── analysis │ ├── addresses.py │ ├── taproot.py │ ├── mining.py │ └── coinbase.py ├── grafana ├── grafana │ ├── Dockerfile │ └── config │ │ └── grafana.ini ├── prometheus │ ├── Dockerfile │ └── config │ │ └── prometheus.yml ├── exporter │ ├── Dockerfile │ ├── btc_conf.py │ └── client.py ├── docker-compose.yml └── readme.md ├── dashboard ├── lightning.gif ├── bitpolito_logo.png ├── pages │ └── Lightning_Network.py └── On-chain.py ├── .env.example ├── tests ├── test_btc_rpc.py ├── test_blockstream.py └── conftest.py ├── Dockerfile ├── pyproject.toml ├── setup.py ├── LICENSE ├── Makefile ├── .gitignore ├── README.md └── requirements.txt /bitdata/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bitdata/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /grafana/grafana/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM grafana/grafana 2 | 3 | ADD config/grafana.ini /etc/grafana/ -------------------------------------------------------------------------------- /grafana/prometheus/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM prom/prometheus 2 | 3 | ADD config/prometheus.yml /etc/prometheus/ 4 | -------------------------------------------------------------------------------- /dashboard/lightning.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitPolito/bitcoin-data-analysis/HEAD/dashboard/lightning.gif -------------------------------------------------------------------------------- /bitdata/provider/__init__.py: -------------------------------------------------------------------------------- 1 | from .bitcoin_rpc import BitcoinRPC 2 | from .blockstream import BlockstreamProvider 3 | -------------------------------------------------------------------------------- /dashboard/bitpolito_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitPolito/bitcoin-data-analysis/HEAD/dashboard/bitpolito_logo.png -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | RPC_USER='bitpolito' 2 | RPC_PASSWORD='bitpolito' 3 | RPC_HOST='localhost' 4 | RPC_PORT=8332 5 | 6 | BOT_TOKEN="XXX" 7 | CHAT_ID="XXX" 8 | DISCORD_WEBHOOK_URL="XXX" 9 | -------------------------------------------------------------------------------- /grafana/exporter/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9 2 | 3 | ADD client.py . 4 | ADD btc_conf.py . 5 | 6 | RUN pip install prometheus_client python-bitcoinlib requests 7 | 8 | CMD ["python", "./client.py"] -------------------------------------------------------------------------------- /grafana/exporter/btc_conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | RPC_SCHEME = "http" 3 | RPC_HOST = "localhost" 4 | RPC_PORT = "8332" 5 | RPC_USER = "Put_here_your_btc_server_username" 6 | RPC_PASSWORD = "put_here_your_btc_server_password" 7 | CONF_PATH = os.environ.get("BITCOIN_CONF_PATH") 8 | 9 | TIMEOUT = 30 -------------------------------------------------------------------------------- /bitdata/core/config.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseSettings 2 | 3 | 4 | class BitConfig(BaseSettings): 5 | RPC_USER: str 6 | RPC_PASSWORD: str 7 | RPC_HOST: str 8 | RPC_PORT: int 9 | 10 | class Config: 11 | env_file = "./.env" 12 | 13 | 14 | cfg = BitConfig() 15 | -------------------------------------------------------------------------------- /tests/test_btc_rpc.py: -------------------------------------------------------------------------------- 1 | from tests.conftest import bitcoin_rpc 2 | 3 | 4 | def test_last_block_height(bitcoin_rpc): 5 | last_block_height = bitcoin_rpc.get_last_block_height() 6 | assert last_block_height is not None 7 | assert isinstance(last_block_height, int) 8 | assert last_block_height > 0 9 | -------------------------------------------------------------------------------- /tests/test_blockstream.py: -------------------------------------------------------------------------------- 1 | from tests.conftest import blockstream_provider 2 | 3 | 4 | def test_get_block(blockstream_provider): 5 | block = blockstream_provider.get_block(0) 6 | assert ( 7 | block["id"] 8 | == "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" 9 | ) 10 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from bitdata.config import BitConfig 4 | from bitdata.provider import BitcoinRPC, BlockstreamProvider 5 | 6 | 7 | @pytest.fixture 8 | def bitcoin_rpc(): 9 | cfg = BitConfig() 10 | return BitcoinRPC(cfg) 11 | 12 | 13 | @pytest.fixture 14 | def blockstream_provider(): 15 | return BlockstreamProvider() 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-slim 2 | RUN apt-get update 3 | RUN apt-get install -y curl python3-dev autoconf 4 | RUN curl -sSL https://install.python-poetry.org | python3 - 5 | ENV PATH="/root/.local/bin:$PATH" 6 | WORKDIR /app 7 | COPY pyproject.toml poetry.lock ./ 8 | RUN poetry config virtualenvs.create false 9 | RUN poetry install --no-dev --no-root 10 | 11 | COPY . . 12 | EXPOSE 8501 13 | 14 | CMD ["poetry", "run", "streamlit", "run", "dashboard/On-chain.py"] -------------------------------------------------------------------------------- /bitdata/notifiers/discord.py: -------------------------------------------------------------------------------- 1 | from discordwebhook import Discord 2 | import os 3 | from dotenv import load_dotenv 4 | 5 | load_dotenv() # take environment variables from .env. 6 | 7 | class DiscordWriter: 8 | def __init__(self): 9 | self.webhookurl = os.getenv("DISCORD_WEBHOOK_URL") 10 | self.discord = Discord(url = self.webhookurl) 11 | 12 | async def send_discord_message(self, message): 13 | self.discord.post(content=message) -------------------------------------------------------------------------------- /bitdata/notifiers/telegram.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dotenv import load_dotenv 4 | from telegram import Bot 5 | from telegram.constants import ParseMode 6 | 7 | load_dotenv() # take environment variables from .env. 8 | 9 | 10 | class TelegramWriter: 11 | def __init__( 12 | self, 13 | ): 14 | self._token = os.getenv("BOT_TOKEN") 15 | self.chat_id = os.getenv("CHAT_ID") 16 | self.bot = Bot(token=self._token) 17 | 18 | async def send_telegram_message(self, message): 19 | if self.bot: 20 | await self.bot.send_message( 21 | chat_id=self.chat_id, text=message, parse_mode=ParseMode.MARKDOWN_V2 22 | ) 23 | -------------------------------------------------------------------------------- /grafana/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | exporter: 3 | build: exporter/. 4 | ports: 5 | - "9000:9000" 6 | networks: 7 | - umbrel_main_networ 8 | restart: on-failure 9 | 10 | prometheus: 11 | build: prometheus/. 12 | ports: 13 | - "9090:9090" 14 | networks: 15 | - umbrel_main_network 16 | restart: on-failure 17 | 18 | grafana: 19 | build: grafana/. 20 | ports: 21 | - "11000:11000" 22 | volumes: 23 | - grafana_data:/var/lib/grafana 24 | - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards 25 | - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources 26 | restart: on-failure 27 | 28 | networks: 29 | umbrel_main_network: 30 | name: umbrel_main_network 31 | external: true 32 | 33 | volumes: 34 | grafana_data: -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "bitdata" 3 | version = "0.1.0" 4 | description = "Tools for analyzing data on Bitcoin and Lightning Network" 5 | authors = ["waltermaffy "] 6 | license = "MIT" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.10" 10 | requests = "^2.31.0" 11 | python-bitcoinrpc = "^1.0" 12 | pandas = "^2.0.2" 13 | loguru = "^0.7.0" 14 | streamlit = "^1.22.0" 15 | pydantic = {extras = ["dotenv"], version = "^1.10.8"} 16 | plotly = "^5.14.1" 17 | Pillow = "^9.5.0" 18 | matplotlib = "^3.7.1" 19 | tqdm = "^4.65.0" 20 | python-telegram-bot = "^20.6" 21 | python-dotenv = "^1.0.0" 22 | discordwebhook = "^1.0.3" 23 | 24 | [tool.poetry.dev-dependencies] 25 | mypy = "^0.971" 26 | black = {version = "^22.8.0", allow-prereleases = true} 27 | isort = "^5.10.1" 28 | pytest-cov = "^4.0.0" 29 | 30 | [build-system] 31 | requires = ["poetry-core>=1.0.0"] 32 | build-backend = "poetry.core.masonry.api" 33 | -------------------------------------------------------------------------------- /bitdata/provider/quiknode.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | 4 | class QuickNode: 5 | def __init__(self, api_url="https://docs-demo.btc.quiknode.pro/"): 6 | self.api_url = api_url 7 | 8 | def decode_raw_transaction(self, raw_transaction_hex): 9 | headers = {"Content-Type": "application/json"} 10 | 11 | # JSON data to be sent in the POST request 12 | data = {"method": "decoderawtransaction", "params": [raw_transaction_hex]} 13 | 14 | try: 15 | response = requests.post(api_url, json=data, headers=headers) 16 | if response.status_code == 200: 17 | decoded_transaction = response.json() 18 | return decoded_transaction 19 | else: 20 | print(f"Error: {response.status_code} - {response.text}") 21 | return None 22 | except Exception as e: 23 | print("Error:", e) 24 | return None 25 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | 3 | import setuptools 4 | 5 | this_directory = path.abspath(path.dirname(__file__)) 6 | with open(path.join(this_directory, "README.md"), encoding="utf-8") as f: 7 | long_description = f.read() 8 | 9 | with open("requirements.txt") as f: 10 | requirements = f.read().splitlines() 11 | 12 | setuptools.setup( 13 | name="bitdata", 14 | version="0.1.0", 15 | description="Tools for analyzing data on Bitcoin and Lightning", 16 | long_description=long_description, 17 | long_description_content_type="text/markdown", 18 | url="https://github.com/BITPoliTO/bitcoin-data-analysis", 19 | author="BitPolito", 20 | author_email="info.bitpolito@protonmail.com", 21 | license="MIT", 22 | packages=setuptools.find_namespace_packages(), 23 | classifiers=[ 24 | "Programming Language :: Python :: 3", 25 | "License :: OSI Approved :: MIT License", 26 | "Operating System :: OS Independent", 27 | ], 28 | python_requires=">=3.7", 29 | install_requires=requirements, 30 | include_package_data=True, 31 | ) 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 BitPolito 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /bitdata/analysis/addresses.py: -------------------------------------------------------------------------------- 1 | from ..provider.bitcoin_rpc import BitcoinRPC 2 | 3 | 4 | def last_block_analytics(rpc_manager: BitcoinRPC): 5 | current_height = rpc_manager.get_last_block_height() 6 | last_block = rpc_manager.get_block_by_height(current_height) 7 | tx_count = len(last_block["tx"]) 8 | # Print the results 9 | print( 10 | "Number of Transactions in Block " + str(current_height) + ": " + str(tx_count) 11 | ) 12 | address_result = rpc_manager.address_block_analytics(last_block) 13 | address_types_count = address_result["address_types_count"] 14 | address_types_amount = address_result["address_types_amount"] 15 | 16 | print("\nNumber of UTXOs by address type:") 17 | for address_type, count in address_types_count.items(): 18 | print(f"{address_type}: {count}") 19 | 20 | print("\nAmount of UTXOs by address type:") 21 | for address_type, amount in address_types_amount.items(): 22 | print(f"{address_type}: {amount}") 23 | 24 | 25 | if __name__ == "__main__": 26 | from ..core.config import BitConfig 27 | 28 | cfg = BitConfig() 29 | rpc_manager = BitcoinRPC(cfg) 30 | last_block_analytics(rpc_manager) 31 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | frontend: 2 | poetry run streamlit run dashboard/On-chain.py 3 | 4 | test: 5 | pytest tests/ 6 | 7 | isort: 8 | poetry run isort --profile black . 9 | 10 | black: 11 | poetry run black . 12 | 13 | format: 14 | make isort 15 | make black 16 | make mypy 17 | 18 | mypy: 19 | poetry run mypy bitdata --ignore-missing 20 | 21 | clean: 22 | rm -r bitdata.egg-info/ || true 23 | find . -name ".DS_Store" -exec rm -f {} \; || true 24 | rm -rf dist || true 25 | rm -rf build || true 26 | 27 | package: 28 | poetry export -f requirements.txt --without-hashes --output requirements.txt 29 | make clean 30 | python setup.py sdist bdist_wheel 31 | 32 | test: 33 | pytest tests/ 34 | 35 | install: 36 | make clean 37 | python setup.py sdist bdist_wheel 38 | pip install --upgrade dist/* 39 | 40 | upload: 41 | make clean 42 | python setup.py sdist bdist_wheel 43 | twine upload --repository pypi dist/* 44 | 45 | docker-build: 46 | docker build -t bitdata . 47 | 48 | docker-run: 49 | docker run -p 8501:8501 -v ./frontend:/app/frontend bitdata 50 | 51 | docker-stop: 52 | -docker stop bitdata 53 | -docker rm bitdata 54 | 55 | docker: 56 | -make docker-stop 57 | -make docker-build 58 | -make docker-run 59 | -------------------------------------------------------------------------------- /grafana/prometheus/config/prometheus.yml: -------------------------------------------------------------------------------- 1 | # my global config 2 | global: 3 | scrape_interval: 150s # Set the scrape interval to every 15 seconds. Default is every 1 minute. 4 | evaluation_interval: 150s # Evaluate rules every 15 seconds. The default is every 1 minute. 5 | # scrape_timeout is set to the global default (10s). 6 | 7 | # Alertmanager configuration 8 | alerting: 9 | alertmanagers: 10 | - static_configs: 11 | - targets: 12 | # - alertmanager:9093 13 | 14 | # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. 15 | rule_files: 16 | # - "first_rules.yml" 17 | # - "second_rules.yml" 18 | 19 | # A scrape configuration containing exactly one endpoint to scrape: 20 | # Here it's Prometheus itself. 21 | scrape_configs: 22 | # The job name is added as a label `job=` to any timeseries scraped from this config. 23 | - job_name: 'prometheus' 24 | 25 | # metrics_path defaults to '/metrics' 26 | # scheme defaults to 'http'. 27 | 28 | static_configs: 29 | - targets: ['localhost:9090'] 30 | # The job name is added as a label `job=` to any timeseries scraped from this config. 31 | - job_name: 'BitPolito-DataAnalysis' 32 | 33 | # metrics_path defaults to '/metrics' 34 | # scheme defaults to 'http'. 35 | 36 | static_configs: 37 | - targets: ['10.0.0.5:9000'] # Change 192.168.1.10 to the IP of local machine -------------------------------------------------------------------------------- /bitdata/provider/blockstream.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | 4 | # Class that use the Blockstream API to get the blockchain data to do analysis 5 | class BlockstreamProvider: 6 | def __init__(self, network="mainnet"): 7 | self.network = network 8 | if network == "testnet": 9 | self.base_url = "https://blockstream.info/testnet/api" 10 | else: 11 | self.base_url = "https://blockstream.info/api" 12 | 13 | def get_block(self, block_height): 14 | return requests.get(f"{self.base_url}/block-height/{block_height}").text 15 | 16 | def get_last_height(self): 17 | return requests.get(f"{self.base_url}/blocks/tip/height").text 18 | 19 | def last_hash(self): 20 | return requests.get(f"{self.base_url}/blocks/tip/hash").text 21 | 22 | def get_last_n_blocks(self, n=10): 23 | result = requests.get(f"{self.base_url}/blocks/:{n}") 24 | print(f"{self.base_url}/blocks/:{n}") 25 | return self.parse_result(result) 26 | 27 | def get_raw_coinbase_transaction(self, block_hash: str = ""): 28 | coinbase_transaction_hash = requests.get( 29 | f"{self.base_url}/block/{block_hash}/txid/0" 30 | ).text 31 | coinbase_transaction_raw = requests.get( 32 | f"{self.base_url}/tx/{coinbase_transaction_hash}/raw" 33 | ).text 34 | return coinbase_transaction_raw 35 | 36 | def parse_result(self, result): 37 | if result.status_code == 200: 38 | return result.json() 39 | else: 40 | return None 41 | 42 | 43 | if __name__ == "__main__": 44 | print("BlockstreamProvider") 45 | # bp = BlockstreamProvider() 46 | 47 | # block = bp.get_block(0) 48 | # print(block) 49 | 50 | # blockchain_info = bp.get_blockchain_info() 51 | # print(blockchain_info) 52 | -------------------------------------------------------------------------------- /grafana/readme.md: -------------------------------------------------------------------------------- 1 | # Dashboard on Grafana 2 | ## Pipeline
3 | - exporter --> prometheus --> grafana
4 | 5 | ## Usage 6 | 1) Edit btc_conf.py in exporter folder, mainly you should change:
7 | - RPC_USER = "Put_here_your_btc_server_username"
8 | - RPC_PASSWORD = "put_here_your_btc_server_password"
9 | - RPC_HOST = "localhost", here you should put the ip of your container (if docker is used)
10 | Others parameters should be ok but take anyway a look.
11 | 12 | 2) In the compose file you probably should change the docker network on which your bitcoin core node is running. If you're using umbrel and your installation of btc core is done through umbrell app store you should already be ok, orherwise change the network to fit your needed. 13 | 14 | 3) Last step is to finally run docker-compose.yml 15 | ``` 16 | docker-compose up --build -d 17 | ``` 18 | As you can see in the compose file, exporter sends data on port 9000, prometheus server is on port 9090 and grafana on 11000.
19 | Compose file also contains a volume for grafana, so you can save your dashboards and data even if you restart the container (prometheus and exporter don't need a volume so far).
20 | To test if everything is working connect on port 11000 on your machine and you should see grafana login page.
21 | 22 | ### Some info on client.py 23 | Client.py is written in python, it's a simple script that connects to bitcoin core node and gets data from it.
24 | If you need debug information just set DEBUG = 1 and you'll see all the data that are fetched from bitcoin core.
25 | At the moment client.py rely on bitcoin core RPC, but it's possible to use bitcoin-cli instead, this feature is available in the code but not used, if needed just change BITCOIN_CLI = 1 and you'll use bitcoin-cli instead of RPC (warning: bitcoin-cli is slower than RPC and you should edit a little bit the code to make it working on your machine ex. right bitcoin-cli path).
-------------------------------------------------------------------------------- /dashboard/pages/Lightning_Network.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | import requests 4 | import streamlit as st 5 | from PIL import Image 6 | 7 | from bitdata.provider.mempool import MempoolProvider 8 | 9 | # ------------- Title of the page ------------- 10 | st.set_page_config( 11 | page_title="Bitcoin Blockchain live analysis", page_icon="₿", layout="wide" 12 | ) 13 | # Title and bitcoin logos. a lot of them. 14 | st.title("Analisi in diretta di Lightning Network - BitPolito") 15 | bitcoin_logo = "https://upload.wikimedia.org/wikipedia/commons/thumb/4/46/Bitcoin.svg/1200px-Bitcoin.svg.png" 16 | bitpolito_logo = Image.open("dashboard/bitpolito_logo.png") 17 | col = st.columns(12) 18 | logos = [bitcoin_logo, bitpolito_logo] * 6 19 | for i in range(12): 20 | col[i].image(logos[i], width=50) 21 | 22 | 23 | # Lightning Network Stats 24 | ln_stats = MempoolProvider().get_lightning_stats() 25 | # ln_sats: id=37293 added='2023-06-02T00:00:00.000Z' channel_count=70378 node_count=15700 total_capacity=536810389159 tor_nodes=11095 clearnet_nodes=2167 unannounced_nodes=1000 avg_capacity=7627531 avg_fee_rate=547 avg_base_fee_mtokens=850 med_capacity=2000000 med_fee_rate=40 med_base_fee_mtokens=125 clearnet_tor_nodes=1438 26 | 27 | st.metric(label="Total number of nodes", value=ln_stats.node_count) 28 | st.metric(label="Total number of channels", value=ln_stats.channel_count) 29 | st.metric(label="Total capacity", value=ln_stats.total_capacity) 30 | st.metric(label="Tor nodes", value=ln_stats.tor_nodes) 31 | st.metric(label="Clearnet nodes", value=ln_stats.clearnet_nodes) 32 | st.metric(label="Unannounced nodes", value=ln_stats.unannounced_nodes) 33 | st.metric(label="Average capacity", value=ln_stats.avg_capacity) 34 | st.metric(label="Average fee rate", value=ln_stats.avg_fee_rate) 35 | 36 | 37 | st.header("Numero totale di nodi e canali") 38 | st.subheader("Numero totale di nodi") 39 | st.write("Il numero totale di nodi è pari a: ") 40 | st.write("Il numero totale di canali è pari a: ") 41 | 42 | # Lightning Graph 43 | 44 | st.header("Lightning Network Graph") 45 | # Add description 46 | st.expander( 47 | """ 48 | Li 49 | """ 50 | ) 51 | 52 | # Add iframe graph 53 | width = 800 54 | height = 600 55 | lngraph = f'