├── .azure-pipelines ├── ci.yml ├── prepare-env.yml └── run-tests.yml ├── .flake8 ├── .gitignore ├── .readthedocs.yaml ├── LICENCE ├── README.rst ├── contrib ├── Dockerfile ├── daemontools │ ├── env │ │ ├── COIN │ │ ├── DAEMON_URL │ │ ├── DB_DIRECTORY │ │ ├── ELECTRUMX │ │ ├── NET │ │ └── USERNAME │ ├── log │ │ └── run │ └── run ├── query.py ├── raspberrypi3 │ ├── install_electrumx.sh │ └── run_electrumx.sh └── systemd │ ├── electrumx.conf │ └── electrumx.service ├── docs ├── ACKNOWLEDGEMENTS ├── HOWTO.rst ├── Makefile ├── PERFORMANCE-NOTES ├── architecture.rst ├── authors.rst ├── changelog.rst ├── conf.py ├── environment.rst ├── features.rst ├── index.rst ├── make.bat ├── peer_discovery.rst ├── protocol-basics.rst ├── protocol-changes.rst ├── protocol-ideas.rst ├── protocol-methods.rst ├── protocol-removed.rst ├── protocol.rst └── rpc-interface.rst ├── electrumx ├── __init__.py ├── lib │ ├── __init__.py │ ├── coins.py │ ├── enum.py │ ├── env_base.py │ ├── hash.py │ ├── merkle.py │ ├── peer.py │ ├── script.py │ ├── server_base.py │ ├── text.py │ ├── tx.py │ └── util.py └── server │ ├── __init__.py │ ├── block_processor.py │ ├── controller.py │ ├── daemon.py │ ├── db.py │ ├── env.py │ ├── history.py │ ├── mempool.py │ ├── peers.py │ ├── session.py │ └── storage.py ├── electrumx_compact_history ├── electrumx_rpc ├── electrumx_server ├── flake8 ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── lib ├── __init__.py ├── test_env_base.py ├── test_hash.py ├── test_merkle.py ├── test_script.py ├── test_tx.py └── test_util.py ├── pytest.ini └── server ├── __init__.py ├── test_api.py ├── test_compaction.py ├── test_daemon.py ├── test_env.py ├── test_mempool.py ├── test_notifications.py ├── test_storage.py └── test_tsc_merkle_proof.py /.azure-pipelines/ci.yml: -------------------------------------------------------------------------------- 1 | pr: 2 | - master 3 | - releases/* 4 | 5 | jobs: 6 | - template: run-tests.yml 7 | parameters: 8 | name: Py38_Ubuntu 9 | vmImage: 'ubuntu-20.04' 10 | pythonVersion: '3.8' 11 | - template: run-tests.yml 12 | parameters: 13 | name: Py38_Mac 14 | vmImage: 'macOS-10.14' 15 | pythonVersion: '3.8' 16 | - template: run-tests.yml 17 | parameters: 18 | name: Py39_Ubuntu 19 | vmImage: 'ubuntu-20.04' 20 | pythonVersion: '3.9' 21 | - template: run-tests.yml 22 | parameters: 23 | name: Py39_Mac 24 | vmImage: 'macOS-10.14' 25 | pythonVersion: '3.9' 26 | -------------------------------------------------------------------------------- /.azure-pipelines/prepare-env.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | onlyPullRequests: false 3 | 4 | steps: 5 | - script: | 6 | python -m pip install aiohttp 7 | python -m pip install aiorpcX 8 | python -m pip install attrs 9 | python -m pip install pylru 10 | python -m pip install uvloop 11 | python -m pip install pycodestyle 12 | python -m pip install coveralls 13 | python -m pip install coverage 14 | python -m pip install pytest 15 | python -m pip install pytest-asyncio 16 | python -m pip install Sphinx 17 | python -m pip install flake8 18 | displayName: Prepare general environment 19 | condition: | 20 | and( 21 | succeeded(), 22 | or( 23 | eq(variables['Build.Reason'], 'PullRequest'), 24 | eq(${{ parameters.onlyPullRequests }}, false) 25 | ) 26 | ) 27 | enabled: true 28 | continueOnError: false 29 | failOnStderr: false 30 | 31 | # Install plyvel depending on platform 32 | - bash: | 33 | python -m pip install plyvel 34 | condition: eq( variables['Agent.OS'], 'Linux' ) 35 | displayName: Install plyvel on Linux 36 | enabled: true 37 | continueOnError: false 38 | failOnStderr: false 39 | - bash: | 40 | brew tap bagonyi/homebrew-formulae git@github.com:bagonyi/homebrew-formulae.git 41 | brew extract --version=1.22 leveldb bagonyi/formulae 42 | brew install leveldb@1.22 43 | pip install plyvel --no-cache-dir 44 | condition: eq( variables['Agent.OS'], 'Darwin' ) 45 | displayName: Install plyvel on MacOS 46 | enabled: true 47 | continueOnError: false 48 | failOnStderr: false 49 | -------------------------------------------------------------------------------- /.azure-pipelines/run-tests.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | name: '' # defaults for any parameters that aren't specified 3 | vmImage: '' 4 | pythonVersion: '' 5 | 6 | jobs: 7 | - job: ${{ parameters.name }} 8 | pool: 9 | vmImage: ${{ parameters.vmImage }} 10 | steps: 11 | - task: UsePythonVersion@0 12 | inputs: 13 | versionSpec: ${{ parameters.pythonVersion }} 14 | addToPath: true 15 | architecture: x64 16 | - template: prepare-env.yml 17 | - script: | 18 | coverage run -m pytest --junitxml=junit/test-results.xml tests 19 | coverage xml 20 | displayName: 'Test with pytest' 21 | - bash: ./flake8 22 | displayName: flake8 23 | - task: PublishTestResults@2 24 | condition: succeededOrFailed() 25 | inputs: 26 | testResultsFiles: 'junit/test-*.xml' 27 | testRunTitle: 'Publish test results for Python ${{ parameters.pythonVersion }}' 28 | - task: PublishCodeCoverageResults@1 29 | inputs: 30 | codeCoverageTool: cobertura 31 | summaryFileLocation: coverage.xml 32 | - bash: | 33 | COVERALLS_REPO_TOKEN=$(COVERALLS_REPO_TOKEN) coveralls 34 | displayName: 'Coveralls' 35 | - bash: | 36 | pycodestyle --max-line-length=100 electrumx/lib/*.py electrumx/server/*.py *.py 37 | condition: succeededOrFailed() 38 | displayName: 'Validate code style' 39 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | 3 | max-line-length=100 4 | select=E,F,W 5 | ignore=W503 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/__pycache__/ 2 | .cache/ 3 | .coverage 4 | .pytest_cache/ 5 | */*~ 6 | *.#* 7 | *# 8 | *~ 9 | docs/_build 10 | /build 11 | /dist 12 | /electrumx.egg-info 13 | .vscode/ 14 | .mypy_cache/ 15 | .idea/ 16 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-20.04 11 | tools: 12 | python: "3.8" 13 | # You can also specify other tool versions: 14 | # nodejs: "16" 15 | # rust: "1.55" 16 | # golang: "1.17" 17 | 18 | # Build documentation in the docs/ directory with Sphinx 19 | sphinx: 20 | configuration: docs/conf.py 21 | 22 | # Optionally build your docs in additional formats such as PDF 23 | # formats: 24 | # - pdf 25 | 26 | python: 27 | install: 28 | - requirements: requirements.txt 29 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-2021, Neil Booth 2 | Copyright (c) 2016-2020, the ElectrumX authors 3 | 4 | All rights reserved. 5 | 6 | Open BSV License version 3 7 | Copyright (c) 2020 Bitcoin Association 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a copy 10 | of this software and associated documentation files (the "Software"), to deal 11 | in the Software without restriction, including without limitation the rights 12 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | copies of the Software, and to permit persons to whom the Software is 14 | furnished to do so, subject to the following conditions: 15 | 16 | 1 - The above copyright notice and this permission notice shall be included in 17 | all copies or substantial portions of the Software. 18 | 2 - The Software, and any software that is derived from the Software or parts thereof, 19 | can only be used on the Bitcoin SV blockchains. The Bitcoin SV blockchains are defined, 20 | for purposes of this license, as the Bitcoin blockchain containing block height #556767 21 | with the hash "000000000000000001d956714215d96ffc00e0afda4cd0a96c96f8d802b1662b" and 22 | that contains the longest persistent chain of blocks that are accepted by the un-modified Software, 23 | as well as the test blockchains that contain blocks that are accepted by the un-modified Software. 24 | 25 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 26 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 28 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 30 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 31 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://travis-ci.org/kyuupichan/electrumx.svg?branch=master 2 | :target: https://travis-ci.org/kyuupichan/electrumx 3 | .. image:: https://coveralls.io/repos/github/kyuupichan/electrumx/badge.svg 4 | :target: https://coveralls.io/github/kyuupichan/electrumx 5 | 6 | =============================================== 7 | ElectrumX - Reimplementation of electrum-server 8 | =============================================== 9 | 10 | For a future network with bigger blocks. 11 | 12 | :Licence: MIT 13 | :Language: Python (>= 3.8) 14 | :Author: Neil Booth 15 | 16 | Documentation 17 | ============= 18 | 19 | See `readthedocs `_. 20 | 21 | 22 | **Neil Booth** kyuupichan@gmail.com https://github.com/kyuupichan 23 | -------------------------------------------------------------------------------- /contrib/Dockerfile: -------------------------------------------------------------------------------- 1 | # example of Dockerfile that builds release of electrumx-1.13.0 2 | # ENV variables can be overrided on the `docker run` command 3 | 4 | FROM ubuntu:18.04 5 | 6 | WORKDIR / 7 | ADD https://github.com/kyuupichan/electrumx/archive/1.13.0.tar.gz / 8 | RUN tar zxvf *.tar.gz 9 | 10 | RUN apt-get update && \ 11 | apt-get -y install python3.7 python3-pip librocksdb-dev libsnappy-dev libbz2-dev libz-dev liblz4-dev && \ 12 | pip3 install aiohttp pylru python-rocksdb 13 | 14 | RUN cd /electrumx* && python3 setup.py install 15 | 16 | ENV SERVICES="tcp://:50001" 17 | ENV COIN=BitcoinSV 18 | ENV DB_DIRECTORY=/db 19 | ENV DAEMON_URL="http://username:password@hostname:port/" 20 | ENV ALLOW_ROOT=true 21 | ENV DB_ENGINE=rocksdb 22 | ENV MAX_SEND=10000000 23 | ENV BANDWIDTH_UNIT_COST=50000 24 | ENV CACHE_MB=2000 25 | 26 | VOLUME /db 27 | 28 | RUN mkdir -p "$DB_DIRECTORY" && ulimit -n 1048576 29 | 30 | CMD ["/usr/bin/python3", "/usr/local/bin/electrumx_server"] 31 | 32 | # build it with eg.: `docker build -t electrumx .` 33 | # run it with eg.: 34 | # `docker run -d --net=host -v /home/electrumx/db/:/db -e DAEMON_URL="http://youruser:yourpass@localhost:8332" -e REPORT_SERVICES=tcp://example.com:50001 electrumx` 35 | # for a proper clean shutdown, send TERM signal to the running container eg.: `docker kill --signal="TERM" CONTAINER_ID` 36 | -------------------------------------------------------------------------------- /contrib/daemontools/env/COIN: -------------------------------------------------------------------------------- 1 | Bitcoin 2 | -------------------------------------------------------------------------------- /contrib/daemontools/env/DAEMON_URL: -------------------------------------------------------------------------------- 1 | http://username:password@host:port/ 2 | -------------------------------------------------------------------------------- /contrib/daemontools/env/DB_DIRECTORY: -------------------------------------------------------------------------------- 1 | /path/to/db/directory 2 | -------------------------------------------------------------------------------- /contrib/daemontools/env/ELECTRUMX: -------------------------------------------------------------------------------- 1 | /path/to/electrumx_server 2 | -------------------------------------------------------------------------------- /contrib/daemontools/env/NET: -------------------------------------------------------------------------------- 1 | mainnet 2 | -------------------------------------------------------------------------------- /contrib/daemontools/env/USERNAME: -------------------------------------------------------------------------------- 1 | electrumx 2 | -------------------------------------------------------------------------------- /contrib/daemontools/log/run: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec multilog s500000 n10 /path/to/log/dir 3 | -------------------------------------------------------------------------------- /contrib/daemontools/run: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | echo "Launching ElectrumX server..." 3 | USERNAME=$(envdir ./env printenv USERNAME) 4 | ELECTRUMX=$(envdir ./env printenv ELECTRUMX) 5 | ulimit -n 4000 && exec 2>&1 envdir ./env envuidgid $USERNAME python3 $ELECTRUMX 6 | -------------------------------------------------------------------------------- /contrib/query.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2016-2018, Neil Booth 4 | # 5 | # All rights reserved. 6 | # 7 | # See the file "LICENCE" for information about the copyright 8 | # and warranty status of this software. 9 | 10 | '''Script to query the database for debugging purposes. 11 | 12 | Not currently documented; might become easier to use in future. 13 | ''' 14 | 15 | import argparse 16 | import asyncio 17 | import sys 18 | 19 | from electrumx import Env 20 | from electrumx.server.db import DB 21 | from electrumx.lib.hash import hash_to_hex_str, Base58Error 22 | 23 | 24 | async def print_stats(hist_db, utxo_db): 25 | count = 0 26 | for key in utxo_db.iterator(prefix=b'u', include_value=False): 27 | count += 1 28 | print(f'UTXO count: {utxos}') 29 | 30 | count = 0 31 | for key in utxo_db.iterator(prefix=b'h', include_value=False): 32 | count += 1 33 | print(f'HashX count: {count}') 34 | 35 | hist = 0 36 | hist_len = 0 37 | for key, value in hist_db.iterator(prefix=b'H'): 38 | hist += 1 39 | hist_len += len(value) // 4 40 | print(f'History rows {hist:,d} entries {hist_len:,d}') 41 | 42 | 43 | def arg_to_hashX(coin, arg): 44 | try: 45 | script = bytes.fromhex(arg) 46 | print(f'Script: {arg}') 47 | return coin.hashX_from_script(script) 48 | except ValueError: 49 | pass 50 | 51 | try: 52 | hashX = coin.address_to_hashX(arg) 53 | print(f'Address: {arg}') 54 | return hashX 55 | except Base58Error: 56 | print(f'Ingoring unknown arg: {arg}') 57 | return None 58 | 59 | 60 | async def query(args): 61 | env = Env() 62 | db = DB(env) 63 | coin = env.coin 64 | 65 | await db.open_for_serving() 66 | 67 | if not args.scripts: 68 | await print_stats(db.hist_db, db.utxo_db) 69 | return 70 | limit = args.limit 71 | for arg in args.scripts: 72 | hashX = arg_to_hashX(coin, arg) 73 | if not hashX: 74 | continue 75 | n = None 76 | history = await db.limited_history(hashX, limit=limit) 77 | for n, (tx_hash, height) in enumerate(history, start=1): 78 | print(f'History #{n:,d}: height {height:,d} ' 79 | f'tx_hash {hash_to_hex_str(tx_hash)}') 80 | if n is None: 81 | print('No history found') 82 | n = None 83 | utxos = await db.all_utxos(hashX) 84 | for n, utxo in enumerate(utxos, start=1): 85 | print(f'UTXO #{n:,d}: tx_hash {hash_to_hex_str(utxo.tx_hash)} ' 86 | f'tx_pos {utxo.tx_pos:,d} height {utxo.height:,d} ' 87 | f'value {utxo.value:,d}') 88 | if n == limit: 89 | break 90 | if n is None: 91 | print('No UTXOs found') 92 | balance = sum(utxo.value for utxo in utxos) 93 | print(f'Balance: {coin.decimal_value(balance):,f} {coin.SHORTNAME}') 94 | 95 | 96 | def main(): 97 | default_limit = 10 98 | parser = argparse.ArgumentParser( 99 | 'query.py', 100 | description='Invoke with COIN and DB_DIRECTORY set in the ' 101 | 'environment as they would be invoking electrumx_server' 102 | ) 103 | parser.add_argument('-l', '--limit', metavar='limit', type=int, 104 | default=10, help=f'maximum number of entries to ' 105 | f'return (default: {default_limit})') 106 | parser.add_argument('scripts', nargs='*', default=[], type=str, 107 | help='hex scripts to query') 108 | args = parser.parse_args() 109 | loop = asyncio.get_event_loop() 110 | loop.run_until_complete(query(args)) 111 | 112 | if __name__ == '__main__': 113 | main() 114 | -------------------------------------------------------------------------------- /contrib/raspberrypi3/install_electrumx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ################### 3 | # install electrumx 4 | ################### 5 | 6 | # Remove "raspi-copies-and-fills" as it breaks the upgrade process 7 | sudo apt-get purge raspi-copies-and-fills 8 | 9 | # upgrade raspbian to 'stretch' distribution 10 | sudo echo 'deb http://mirrordirector.raspbian.org/raspbian/ testing main contrib non-free rpi' > /etc/apt/sources.list.d/stretch.list 11 | sudo apt-get update 12 | sudo apt-get dist-upgrade 13 | sudo apt-get autoremove 14 | 15 | # install electrumx dependencies 16 | sudo apt-get install python3-pip 17 | sudo apt-get install build-essential libc6-dev 18 | sudo apt-get install libncurses5-dev libncursesw5-dev 19 | sudo apt install libreadline6-dev/stable libreadline6/stable 20 | sudo apt-get install libleveldb-dev 21 | sudo apt-get install git 22 | sudo pip3 install plyvel 23 | 24 | # install electrumx 25 | git clone https://github.com/kyuupichan/electrumx.git 26 | cd electrumx 27 | sudo python3 setup.py install 28 | -------------------------------------------------------------------------------- /contrib/raspberrypi3/run_electrumx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ############### 3 | # run_electrumx 4 | ############### 5 | 6 | # configure electrumx 7 | export COIN=BitcoinSegwit 8 | export DAEMON_URL=http://rpcuser:rpcpassword@127.0.0.1 9 | export NET=mainnet 10 | export CACHE_MB=400 11 | export DB_DIRECTORY=/home/username/.electrumx/db 12 | export SSL_CERTFILE=/home/username/.electrumx/certfile.crt 13 | export SSL_KEYFILE=/home/username/.electrumx/keyfile.key 14 | export BANNER_FILE=/home/username/.electrumx/banner 15 | export DONATION_ADDRESS=your-donation-address 16 | 17 | # connectivity 18 | export HOST= 19 | export TCP_PORT=50001 20 | export SSL_PORT=50002 21 | 22 | # visibility 23 | export REPORT_HOST=hostname.com 24 | export RPC_PORT=8000 25 | 26 | # run electrumx 27 | ulimit -n 10000 28 | /usr/local/bin/electrumx_server 2>> /home/username/.electrumx/electrumx.log >> /home/username/.electrumx/electrumx.log & 29 | 30 | ###################### 31 | # auto-start electrumx 32 | ###################### 33 | 34 | # add this line to crontab -e 35 | # @reboot /path/to/run_electrumx.sh 36 | -------------------------------------------------------------------------------- /contrib/systemd/electrumx.conf: -------------------------------------------------------------------------------- 1 | # default /etc/electrumx.conf for systemd 2 | 3 | # REQUIRED 4 | DB_DIRECTORY = /db 5 | # Bitcoin Node RPC Credentials 6 | DAEMON_URL = http://username:password@hostname:port/ 7 | 8 | # COIN = BitcoinSegwit 9 | 10 | # See http://electrumx.readthedocs.io/en/latest/environment.html for 11 | # information about other configuration settings you probably want to consider. 12 | -------------------------------------------------------------------------------- /contrib/systemd/electrumx.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Electrumx 3 | After=network.target 4 | 5 | [Service] 6 | EnvironmentFile=/etc/electrumx.conf 7 | ExecStart=/usr/local/bin/electrumx_server 8 | User=electrumx 9 | LimitNOFILE=8192 10 | TimeoutStopSec=30min 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /docs/ACKNOWLEDGEMENTS: -------------------------------------------------------------------------------- 1 | Thanks to Thomas Voegtlin for creating the Electrum software and 2 | infrastructure and for maintaining it so diligently. Electrum is 3 | probably the best desktop Bitcoin wallet solution for most users. My 4 | faith in it is such that I use Electrum software to store most of my 5 | Bitcoins. 6 | 7 | Whilst the vast majority of the code here is my own original work and 8 | includes some new ideas, it is very clear that the general structure 9 | and concept are those of Electrum. Some parts of the code and ideas 10 | of Electrum, some of which it itself took from other projects such as 11 | Abe and pywallet, remain. Thanks to the authors of all the software 12 | this is derived from. 13 | 14 | I'd like to thank bauerj, hsmiths and JWU42 of #electrum for their 15 | help and support in testing the versions of ElectrumX prior to 1.0. 16 | 17 | Thanks to Daniel Bernstein for daemontools and other software, and to 18 | Matthew Dillon for DragonFlyBSD. They are both deeply inspirational 19 | people. 20 | 21 | And of course, thanks to Satoshi for the wonderful creation that is 22 | Bitcoin. -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = -W 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = ElectrumX 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/PERFORMANCE-NOTES: -------------------------------------------------------------------------------- 1 | Just some notes on performance with Python 3.5. We are taking these into 2 | account in the code. 3 | 4 | - 60% faster to create lists with [] list comprehensions than tuples 5 | or lists with tuple(), list(). Of those list is 10% faster than 6 | tuple. 7 | 8 | - however when not initializing from a generator, a fixed-length tuple 9 | is at least 80% faster than a list. 10 | 11 | - an implicit default argument is ~5% faster than passing the default 12 | explicitly 13 | 14 | - using a local variable x rather than self.x in loops and list 15 | comprehensions is over 50% faster 16 | 17 | - struct.pack, struct.unpack are over 60% faster than int.to_bytes and 18 | int.from_bytes. They are faster little endian (presumably because 19 | it matches the host) than big endian regardless of length. Furthermore, 20 | using stored packing and unpacking methods from Struct classes is faster 21 | than using the flexible-format struct.[un]pack equivalents. 22 | 23 | After storing the Struct('`_. and uses 18 | Azure Pipelines for Continuous Integration. 19 | 20 | Please submit an issue on the `bug tracker 21 | `_ if you have found a 22 | bug or have a suggestion to improve the server. 23 | 24 | Authors and License 25 | =================== 26 | 27 | Neil Booth wrote the vast majority of the code; see :ref:`Authors`. 28 | Python version at least 3.8 is required. 29 | 30 | The code is released under the `MIT Licence 31 | `_. 32 | 33 | Getting Started 34 | =============== 35 | 36 | See :ref:`HOWTO`. 37 | 38 | There is also an `installer`_ available that simplifies the 39 | installation on various Linux-based distributions, and a `Dockerfile`_ 40 | available . 41 | 42 | .. _installer: https://github.com/bauerj/electrumx-installer 43 | .. _Dockerfile: https://github.com/lukechilds/docker-electrumx 44 | 45 | Documentation 46 | ============= 47 | 48 | .. toctree:: 49 | 50 | features 51 | changelog 52 | HOWTO 53 | environment 54 | protocol 55 | peer_discovery 56 | rpc-interface 57 | architecture 58 | authors 59 | 60 | Indices and tables 61 | ================== 62 | 63 | * :ref:`genindex` 64 | * :ref:`search` 65 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=ElectrumX 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/peer_discovery.rst: -------------------------------------------------------------------------------- 1 | .. _Peer Discovery: 2 | 3 | Peer Discovery 4 | ============== 5 | 6 | This was implemented in ElectrumX as of version 0.11.0. Support for 7 | IRC peer discovery was removed in ElectrumX version 1.2.1. 8 | 9 | The :dfn:`peer database` is an in-memory store of peers with at least 10 | the following information about a peer, required for a response to the 11 | :func:`server.peers.subscribe` RPC call: 12 | 13 | * host name 14 | * ip address 15 | * TCP and SSL port numbers 16 | * protocol version 17 | * pruning limit, if any 18 | 19 | 20 | Hard-coded Peers 21 | ---------------- 22 | 23 | A list of hard-coded, well-known peers seeds the peer discovery 24 | process. Ideally it should have at least 4 servers that have shown 25 | commitment to reliable service. 26 | 27 | In ElectrumX this is a per-coin property in `lib/coins.py 28 | `_. 29 | 30 | 31 | server.peers.subscribe 32 | ---------------------- 33 | 34 | :func:`server.peers.subscribe` is used by Electrum clients to get a 35 | list of peer servers, in preference to a hard-coded list of peer 36 | servers in the client, which it will fall back to if necessary. 37 | 38 | The server should craft its response in a way that reduces the 39 | effectiveness of server sybil attacks and peer spamming. 40 | 41 | The response should only include peers it has successfully connected 42 | to recently. Only reporting recent good peers ensures that those that 43 | have gone offline will be forgotten quickly and not be passed around 44 | for long. 45 | 46 | In ElectrumX, "recently" is taken to be the last 24 hours. Only one 47 | peer from each IPv4/16 netmask is returned, and the number of onion 48 | peers is limited. 49 | 50 | 51 | Maintaining the Peer Database 52 | ----------------------------- 53 | 54 | In order to keep its peer database up-to-date and fresh, after some 55 | time has passed since the last successful connection to a peer, an 56 | Electrum server should make another attempt to connect, choosing 57 | either the TCP or SSL port. 58 | 59 | On connecting it should issue :func:`server.peers.subscribe`, 60 | :func:`blockchain.headers.subscribe`, and :func:`server.features` RPC 61 | calls to collect information about the server and its peers. If the 62 | peer seems to not know of you, you can issue a :func:`server.add_peer` 63 | call to advertise yourself. Once this is done and replies received, 64 | terminate the connection. 65 | 66 | The peer database should view information obtained from an outgoing 67 | connection as authoritative, and prefer it to information obtained 68 | from any other source. 69 | 70 | On connecting, a server should confirm the peer is serving the same 71 | network, ideally via the genesis block hash of the 72 | :func:`server.features` RPC call below. Also the height reported by 73 | the peer should be within a small number of the expected value. If a 74 | peer is on the wrong network it should never be advertised to clients 75 | or other peers. Such invalid peers should perhaps be remembered for a 76 | short time to prevent redundant revalidation if other peers persist in 77 | advertising them, and later forgotten. 78 | 79 | If a connection attempt fails, subsequent reconnection attempts should 80 | follow some kind of exponential backoff. 81 | 82 | If a long period of time has elapsed since the last successful 83 | connection attempt, the peer entry should be removed from the 84 | database. This ensures that all peers that have gone offline will 85 | eventually be forgotten by the network entirely. 86 | 87 | ElectrumX will connect to the SSL port if both ports are available. 88 | If that fails it will fall back to the TCP port. It tries to 89 | reconnect to a good peer at least once every 24 hours, and a failing 90 | after 5 minutes but with exponential backoff. It forgets a peer 91 | entirely if a few days have passed since a successful connection. 92 | ElectrumX attempts to connect to onion peers through a Tor proxy that 93 | can be configured or that it will try to autodetect. 94 | 95 | 96 | server.features 97 | --------------- 98 | 99 | :func:`server.features` is a fairly new RPC call that a server can use 100 | to advertise what services and features it offers. It is intended for 101 | use by Electrum clients as well as other peers. Peers will use it to 102 | gather peer information from the peer itself. 103 | 104 | The call takes no arguments and returns a dictionary keyed by feature 105 | name whose value gives details about the feature where appropriate. 106 | If a key is missing the feature is presumed not to be offered. 107 | 108 | 109 | server.add_peer 110 | --------------- 111 | 112 | :func:`server.add_peer` is intended for a new server to get itself in 113 | the connected set. 114 | 115 | A server receiving a :func:`server.add_peer` call should not replace 116 | existing information about the host(s) given, but instead schedule a 117 | separate connection to verify the information for itself. 118 | 119 | To prevent abuse a server may do nothing with second and subsequent 120 | calls to this method from a single connection. 121 | 122 | The result should be True if accepted and False otherwise. 123 | 124 | 125 | Notes for Implementors 126 | ---------------------- 127 | 128 | * it is very important to only accept peers that appear to be on the 129 | same network. At a minimum the genesis hash should be compared (if 130 | the peer supports :func:`server.features`), and also that the peer's 131 | reported height is within a few blocks of your own server's height. 132 | * care should be taken with the :func:`server.add_peer` call. 133 | Consider only accepting it once per connection. Clearnet peer 134 | requests should check the peer resolves to the requesting IP 135 | address, to prevent attackers from being able to trigger arbitrary 136 | outgoing connections from your server. This doesn't work for onion 137 | peers so they should be rate-limited. 138 | * it should be possible for a peer to change their port assignments - 139 | presumably connecting to the old ports to perform checks will not 140 | work. 141 | * peer host names should be checked for validity before accepting 142 | them; and `localhost` should probably be rejected. If it is an IP 143 | address it should be a normal public one (not private, multicast or 144 | unspecified). 145 | * you should limit the number of new peers accepted from any single 146 | source to at most a handful, to limit the effectiveness of malicious 147 | peers wanting to trigger arbitrary outgoing connections or fill your 148 | peer tables with junk data. 149 | * in the response to :func:`server.peers.subscribe` calls, consider 150 | limiting the number of peers on similar IP subnets to protect 151 | against sybil attacks, and in the case of onion servers the total 152 | returned. 153 | * you should not advertise a peer's IP address if it also advertises a 154 | hostname (avoiding duplicates). 155 | -------------------------------------------------------------------------------- /docs/protocol-basics.rst: -------------------------------------------------------------------------------- 1 | Protocol Basics 2 | =============== 3 | 4 | Message Stream 5 | -------------- 6 | 7 | Clients and servers communicate using **JSON RPC** over an unspecified underlying stream 8 | transport. Examples include TCP, SSL, WS and WSS. 9 | 10 | Two standards `JSON RPC 1.0 11 | `_ and `JSON RPC 2.0 12 | `_ are specified; use of version 13 | 2.0 is encouraged but not required. Server support of batch requests 14 | is encouraged for version 1.0 but not required. 15 | 16 | .. note:: A client or server should only indicate JSON RPC 2.0 by 17 | setting the `jsonrpc 18 | `_ member of 19 | its messages to ``"2.0"`` if it supports the version 2.0 protocol in 20 | its entirety. ElectrumX does and will expect clients advertizing so 21 | to function correctly. Those that do not will be disconnected and 22 | possibly blacklisted. 23 | 24 | Clients making batch requests should limit their size depending on the 25 | nature of their query, because servers will limit response size as an 26 | anti-DoS mechanism. 27 | 28 | Over TCP and SSL raw sockets each RPC call, and each response, MUST be terminated by a 29 | single newline to delimit messages. Websocket messages are already framed so they MUST 30 | NOT be newline terminated. The JSON specification does not permit control characters 31 | within strings, so no confusion is possible there. However it does permit newlines as 32 | extraneous whitespace between elements; client and server MUST NOT use newlines in such a 33 | way. 34 | 35 | If using JSON RPC 2.0's feature of parameter passing by name, the 36 | names shown in the description of the method or notification in 37 | question MUST be used. 38 | 39 | A server advertising support for a particular protocol version MUST 40 | support each method documented for that protocol version, unless the 41 | method is explicitly marked optional. It may support other methods or 42 | additional parameters with unspecified behaviour. Use of additional 43 | parameters is discouraged as it may conflict with future versions of 44 | the protocol. 45 | 46 | 47 | Notifications 48 | ------------- 49 | 50 | Some RPC calls are subscriptions which, after the initial response, 51 | will send a JSON RPC :dfn:`notification` each time the thing 52 | subscribed to changes. The `method` of the notification is the same 53 | as the method of the subscription, and the `params` of the 54 | notification (and their names) are given in the documentation of the 55 | method. 56 | 57 | 58 | Version Negotiation 59 | ------------------- 60 | 61 | It is desirable to have a way to enhance and improve the protocol 62 | without forcing servers and clients to upgrade at the same time. 63 | 64 | Protocol versions are denoted by dotted number strings with at least 65 | one dot. Examples: "1.5", "1.4.1", "2.0". In "a.b.c" *a* is the 66 | major version number, *b* the minor version number, and *c* the 67 | revision number. 68 | 69 | A party to a connection will speak all protocol versions in a range, 70 | say from `protocol_min` to `protocol_max`, which may be the same. 71 | When a connection is made, both client and server must initially 72 | assume the protocol to use is their own `protocol_min`. 73 | 74 | The client should send a :func:`server.version` RPC call as early as 75 | possible in order to negotiate the precise protocol version; see its 76 | description for more detail. All responses received in the stream 77 | from and including the server's response to this call will use its 78 | negotiated protocol version. 79 | 80 | 81 | .. _script hashes: 82 | 83 | Script Hashes 84 | ------------- 85 | 86 | A :dfn:`script hash` is the hash of the binary bytes of the locking 87 | script (ScriptPubKey), expressed as a hexadecimal string. The hash 88 | function to use is given by the "hash_function" member of 89 | :func:`server.features` (currently :func:`sha256` only). Like for 90 | block and transaction hashes, when converting the big-endian binary 91 | hash to a hexadecimal string the least-significant byte appears first, 92 | and the most-significant byte last. 93 | 94 | For example, the legacy Bitcoin address from the genesis block:: 95 | 96 | 1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa 97 | 98 | has P2PKH script:: 99 | 100 | 76a91462e907b15cbf27d5425399ebf6f0fb50ebb88f1888ac 101 | 102 | with SHA256 hash:: 103 | 104 | 6191c3b590bfcfa0475e877c302da1e323497acf3b42c08d8fa28e364edf018b 105 | 106 | which is sent to the server reversed as:: 107 | 108 | 8b01df4e368ea28f8dc0423bcf7a4923e3a12d307c875e47a0cfbf90b5c39161 109 | 110 | By subscribing to this hash you can find P2PKH payments to that address. 111 | 112 | One public key, the genesis block public key, among the trillions for 113 | that address is:: 114 | 115 | 04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb 116 | 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f 117 | 118 | which has P2PK script:: 119 | 120 | 4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb 121 | 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac 122 | 123 | with SHA256 hash:: 124 | 125 | 3318537dfb3135df9f3d950dbdf8a7ae68dd7c7dfef61ed17963ff80f3850474 126 | 127 | which is sent to the server reversed as:: 128 | 129 | 740485f380ff6379d11ef6fe7d7cdd68aea7f8bd0d953d9fdf3531fb7d531833 130 | 131 | By subscribing to this hash you can find P2PK payments to the genesis 132 | block public key. 133 | 134 | .. note:: The Genesis block coinbase is uniquely unspendable and 135 | therefore not indexed. It will not show with the above P2PK script 136 | hash subscription. 137 | 138 | 139 | .. _status: 140 | 141 | Status 142 | ------ 143 | 144 | To calculate the `status` of a :ref:`script hash