├── .dockerignore ├── .github └── workflows │ ├── ci.yml │ └── tag.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── Dockerfile.ci ├── LICENSE ├── README.md ├── bindex-cli ├── Cargo.toml └── src │ └── bin │ └── bindex-cli.rs ├── bindex-lib ├── Cargo.lock ├── Cargo.toml └── src │ ├── address │ ├── cache.rs │ └── mod.rs │ ├── chain.rs │ ├── cli.rs │ ├── client.rs │ ├── db.rs │ ├── index.rs │ └── lib.rs ├── electrum ├── LICENCE ├── __init__.py ├── merkle.py └── server.py └── run.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | .* 2 | _* 3 | contrib 4 | /db* 5 | Dockerfile 6 | target 7 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | schedule: 9 | - cron: '0 0 * * *' 10 | 11 | env: 12 | CARGO_TERM_COLOR: always 13 | 14 | jobs: 15 | ubuntu_noble: 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | - uses: dtolnay/rust-toolchain@stable 21 | with: 22 | components: rustfmt, clippy 23 | 24 | - name: Install Rust 25 | run: rustup component add rustfmt clippy 26 | 27 | - name: Install other dependencies 28 | run: sudo apt-get -qqy install build-essential libclang-dev 29 | 30 | - uses: actions/cache@v4 31 | with: 32 | path: | 33 | ~/.cargo/bin/ 34 | ~/.cargo/registry/index/ 35 | ~/.cargo/registry/cache/ 36 | ~/.cargo/git/db/ 37 | target/ 38 | key: cargo-release-${{ hashFiles('**/Cargo.lock') }}-${{ runner.os }} 39 | 40 | - name: Dependency tree 41 | run: cargo tree --locked 42 | 43 | - name: Build 44 | run: | 45 | cargo build --release --all --locked --timings 46 | mv -v target/cargo-timings/cargo-timing-*.html . 47 | mv -v target/release/bindex-cli /usr/local/bin/ 48 | 49 | - name: Sanity 50 | run: | 51 | bindex-cli --version # make sure it can run 52 | bindex-cli --help 53 | 54 | - name: Test 55 | run: cargo test --release --all --locked 56 | 57 | - name: Format 58 | run: cargo fmt --all -- --check 59 | 60 | - name: Clippy 61 | run: cargo clippy --release --all --locked -- -D warnings 62 | 63 | - uses: actions/upload-artifact@v4 64 | with: 65 | name: cargo-build-timings 66 | path: cargo-timing-*.html 67 | retention-days: 30 68 | 69 | 70 | debian_trixie: 71 | runs-on: ubuntu-latest 72 | 73 | steps: 74 | - name: Checkout 75 | uses: actions/checkout@v4 76 | - name: Build 77 | run: docker build -f Dockerfile.ci . -t bindex:latest 78 | - name: Sanity 79 | run: | 80 | docker run --rm bindex:latest bindex-cli --version 81 | docker run --rm bindex:latest bindex-cli --help 82 | 83 | python: 84 | runs-on: ubuntu-latest 85 | 86 | steps: 87 | - uses: actions/checkout@v4 88 | - name: Install Python 89 | uses: actions/setup-python@v5 90 | with: 91 | python-version: "3.11" 92 | - name: Install dependencies 93 | run: | 94 | python -m pip install --upgrade pip 95 | pip install ruff 96 | # Update output format to enable automatic inline annotations. 97 | - name: Ruff check 98 | run: ruff check --output-format=github . 99 | - name: Ruff format 100 | run: ruff format --diff 101 | -------------------------------------------------------------------------------- /.github/workflows/tag.yml: -------------------------------------------------------------------------------- 1 | name: Tag 2 | 3 | on: 4 | push: 5 | tags: [ "*" ] 6 | 7 | env: 8 | CARGO_TERM_COLOR: always 9 | 10 | jobs: 11 | ubuntu_noble: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | - uses: dtolnay/rust-toolchain@stable 17 | 18 | - name: Install Rust 19 | run: rustup component add rustfmt clippy 20 | 21 | - name: Install other dependencies 22 | run: sudo apt-get -qqy install build-essential libclang-dev 23 | 24 | - name: Publish Dry-Run 25 | run: | 26 | cargo publish --dry-run -p bindex 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Rust 2 | /debug 3 | /target 4 | /db 5 | *.log 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | cover/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | .pybuilder/ 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | # For a library or package, you might want to ignore these files since the code is 93 | # intended to run in multiple environments; otherwise, check them in: 94 | # .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # UV 104 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | #uv.lock 108 | 109 | # poetry 110 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 111 | # This is especially recommended for binary packages to ensure reproducibility, and is more 112 | # commonly ignored for libraries. 113 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 114 | #poetry.lock 115 | 116 | # pdm 117 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 118 | #pdm.lock 119 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 120 | # in version control. 121 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 122 | .pdm.toml 123 | .pdm-python 124 | .pdm-build/ 125 | 126 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 127 | __pypackages__/ 128 | 129 | # Celery stuff 130 | celerybeat-schedule 131 | celerybeat.pid 132 | 133 | # SageMath parsed files 134 | *.sage.py 135 | 136 | # Environments 137 | .env 138 | .venv 139 | env/ 140 | venv/ 141 | ENV/ 142 | env.bak/ 143 | venv.bak/ 144 | 145 | # Spyder project settings 146 | .spyderproject 147 | .spyproject 148 | 149 | # Rope project settings 150 | .ropeproject 151 | 152 | # mkdocs documentation 153 | /site 154 | 155 | # mypy 156 | .mypy_cache/ 157 | .dmypy.json 158 | dmypy.json 159 | 160 | # Pyre type checker 161 | .pyre/ 162 | 163 | # pytype static type analyzer 164 | .pytype/ 165 | 166 | # Cython debug symbols 167 | cython_debug/ 168 | 169 | # PyCharm 170 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 171 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 172 | # and can be added to the global gitignore or merged into this file. For a more nuclear 173 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 174 | #.idea/ 175 | 176 | # Ruff stuff: 177 | .ruff_cache/ 178 | 179 | # PyPI configuration file 180 | .pypirc 181 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 4 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "1.1.3" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "anstream" 16 | version = "0.6.18" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" 19 | dependencies = [ 20 | "anstyle", 21 | "anstyle-parse", 22 | "anstyle-query", 23 | "anstyle-wincon", 24 | "colorchoice", 25 | "is_terminal_polyfill", 26 | "utf8parse", 27 | ] 28 | 29 | [[package]] 30 | name = "anstyle" 31 | version = "1.0.10" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" 34 | 35 | [[package]] 36 | name = "anstyle-parse" 37 | version = "0.2.6" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" 40 | dependencies = [ 41 | "utf8parse", 42 | ] 43 | 44 | [[package]] 45 | name = "anstyle-query" 46 | version = "1.1.2" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" 49 | dependencies = [ 50 | "windows-sys", 51 | ] 52 | 53 | [[package]] 54 | name = "anstyle-wincon" 55 | version = "3.0.7" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" 58 | dependencies = [ 59 | "anstyle", 60 | "once_cell", 61 | "windows-sys", 62 | ] 63 | 64 | [[package]] 65 | name = "arrayvec" 66 | version = "0.7.6" 67 | source = "registry+https://github.com/rust-lang/crates.io-index" 68 | checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" 69 | 70 | [[package]] 71 | name = "autocfg" 72 | version = "1.4.0" 73 | source = "registry+https://github.com/rust-lang/crates.io-index" 74 | checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" 75 | 76 | [[package]] 77 | name = "base58ck" 78 | version = "0.1.0" 79 | source = "registry+https://github.com/rust-lang/crates.io-index" 80 | checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f" 81 | dependencies = [ 82 | "bitcoin-internals", 83 | "bitcoin_hashes", 84 | ] 85 | 86 | [[package]] 87 | name = "base64" 88 | version = "0.22.1" 89 | source = "registry+https://github.com/rust-lang/crates.io-index" 90 | checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" 91 | 92 | [[package]] 93 | name = "bech32" 94 | version = "0.11.0" 95 | source = "registry+https://github.com/rust-lang/crates.io-index" 96 | checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" 97 | 98 | [[package]] 99 | name = "bindex" 100 | version = "0.0.11" 101 | dependencies = [ 102 | "bitcoin", 103 | "bitcoin_slices", 104 | "clap", 105 | "hex", 106 | "hex_lit", 107 | "log", 108 | "rocksdb", 109 | "rusqlite", 110 | "thiserror", 111 | "ureq", 112 | ] 113 | 114 | [[package]] 115 | name = "bindex-cli" 116 | version = "0.0.11" 117 | dependencies = [ 118 | "bindex", 119 | "chrono", 120 | "clap", 121 | "env_logger", 122 | "log", 123 | "rusqlite", 124 | "tabled", 125 | ] 126 | 127 | [[package]] 128 | name = "bindgen" 129 | version = "0.69.5" 130 | source = "registry+https://github.com/rust-lang/crates.io-index" 131 | checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" 132 | dependencies = [ 133 | "bitflags", 134 | "cexpr", 135 | "clang-sys", 136 | "itertools", 137 | "lazy_static", 138 | "lazycell", 139 | "proc-macro2", 140 | "quote", 141 | "regex", 142 | "rustc-hash", 143 | "shlex", 144 | "syn", 145 | ] 146 | 147 | [[package]] 148 | name = "bitcoin" 149 | version = "0.32.5" 150 | source = "registry+https://github.com/rust-lang/crates.io-index" 151 | checksum = "ce6bc65742dea50536e35ad42492b234c27904a27f0abdcbce605015cb4ea026" 152 | dependencies = [ 153 | "base58ck", 154 | "bech32", 155 | "bitcoin-internals", 156 | "bitcoin-io", 157 | "bitcoin-units", 158 | "bitcoin_hashes", 159 | "hex-conservative", 160 | "hex_lit", 161 | "secp256k1", 162 | ] 163 | 164 | [[package]] 165 | name = "bitcoin-internals" 166 | version = "0.3.0" 167 | source = "registry+https://github.com/rust-lang/crates.io-index" 168 | checksum = "30bdbe14aa07b06e6cfeffc529a1f099e5fbe249524f8125358604df99a4bed2" 169 | 170 | [[package]] 171 | name = "bitcoin-io" 172 | version = "0.1.3" 173 | source = "registry+https://github.com/rust-lang/crates.io-index" 174 | checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf" 175 | 176 | [[package]] 177 | name = "bitcoin-units" 178 | version = "0.1.2" 179 | source = "registry+https://github.com/rust-lang/crates.io-index" 180 | checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2" 181 | dependencies = [ 182 | "bitcoin-internals", 183 | ] 184 | 185 | [[package]] 186 | name = "bitcoin_hashes" 187 | version = "0.14.0" 188 | source = "registry+https://github.com/rust-lang/crates.io-index" 189 | checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16" 190 | dependencies = [ 191 | "bitcoin-io", 192 | "hex-conservative", 193 | ] 194 | 195 | [[package]] 196 | name = "bitcoin_slices" 197 | version = "0.10.0" 198 | source = "registry+https://github.com/rust-lang/crates.io-index" 199 | checksum = "7943d4257fdfc85afe2a13d2b539a5213e97bb978329dd79b624acbce73042fb" 200 | dependencies = [ 201 | "bitcoin", 202 | "bitcoin_hashes", 203 | ] 204 | 205 | [[package]] 206 | name = "bitflags" 207 | version = "2.9.0" 208 | source = "registry+https://github.com/rust-lang/crates.io-index" 209 | checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" 210 | 211 | [[package]] 212 | name = "bytecount" 213 | version = "0.6.8" 214 | source = "registry+https://github.com/rust-lang/crates.io-index" 215 | checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" 216 | 217 | [[package]] 218 | name = "bytes" 219 | version = "1.10.1" 220 | source = "registry+https://github.com/rust-lang/crates.io-index" 221 | checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" 222 | 223 | [[package]] 224 | name = "bzip2-sys" 225 | version = "0.1.13+1.0.8" 226 | source = "registry+https://github.com/rust-lang/crates.io-index" 227 | checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" 228 | dependencies = [ 229 | "cc", 230 | "pkg-config", 231 | ] 232 | 233 | [[package]] 234 | name = "cc" 235 | version = "1.2.16" 236 | source = "registry+https://github.com/rust-lang/crates.io-index" 237 | checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" 238 | dependencies = [ 239 | "jobserver", 240 | "libc", 241 | "shlex", 242 | ] 243 | 244 | [[package]] 245 | name = "cexpr" 246 | version = "0.6.0" 247 | source = "registry+https://github.com/rust-lang/crates.io-index" 248 | checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" 249 | dependencies = [ 250 | "nom", 251 | ] 252 | 253 | [[package]] 254 | name = "chrono" 255 | version = "0.4.40" 256 | source = "registry+https://github.com/rust-lang/crates.io-index" 257 | checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" 258 | dependencies = [ 259 | "num-traits", 260 | ] 261 | 262 | [[package]] 263 | name = "clang-sys" 264 | version = "1.8.1" 265 | source = "registry+https://github.com/rust-lang/crates.io-index" 266 | checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" 267 | dependencies = [ 268 | "glob", 269 | "libc", 270 | ] 271 | 272 | [[package]] 273 | name = "clap" 274 | version = "4.5.31" 275 | source = "registry+https://github.com/rust-lang/crates.io-index" 276 | checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" 277 | dependencies = [ 278 | "clap_builder", 279 | "clap_derive", 280 | ] 281 | 282 | [[package]] 283 | name = "clap_builder" 284 | version = "4.5.31" 285 | source = "registry+https://github.com/rust-lang/crates.io-index" 286 | checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" 287 | dependencies = [ 288 | "anstream", 289 | "anstyle", 290 | "clap_lex", 291 | "strsim", 292 | ] 293 | 294 | [[package]] 295 | name = "clap_derive" 296 | version = "4.5.28" 297 | source = "registry+https://github.com/rust-lang/crates.io-index" 298 | checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" 299 | dependencies = [ 300 | "heck", 301 | "proc-macro2", 302 | "quote", 303 | "syn", 304 | ] 305 | 306 | [[package]] 307 | name = "clap_lex" 308 | version = "0.7.4" 309 | source = "registry+https://github.com/rust-lang/crates.io-index" 310 | checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" 311 | 312 | [[package]] 313 | name = "colorchoice" 314 | version = "1.0.3" 315 | source = "registry+https://github.com/rust-lang/crates.io-index" 316 | checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" 317 | 318 | [[package]] 319 | name = "either" 320 | version = "1.15.0" 321 | source = "registry+https://github.com/rust-lang/crates.io-index" 322 | checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" 323 | 324 | [[package]] 325 | name = "env_filter" 326 | version = "0.1.3" 327 | source = "registry+https://github.com/rust-lang/crates.io-index" 328 | checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" 329 | dependencies = [ 330 | "log", 331 | "regex", 332 | ] 333 | 334 | [[package]] 335 | name = "env_logger" 336 | version = "0.11.6" 337 | source = "registry+https://github.com/rust-lang/crates.io-index" 338 | checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" 339 | dependencies = [ 340 | "anstream", 341 | "anstyle", 342 | "env_filter", 343 | "humantime", 344 | "log", 345 | ] 346 | 347 | [[package]] 348 | name = "fallible-iterator" 349 | version = "0.3.0" 350 | source = "registry+https://github.com/rust-lang/crates.io-index" 351 | checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" 352 | 353 | [[package]] 354 | name = "fallible-streaming-iterator" 355 | version = "0.1.9" 356 | source = "registry+https://github.com/rust-lang/crates.io-index" 357 | checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" 358 | 359 | [[package]] 360 | name = "fnv" 361 | version = "1.0.7" 362 | source = "registry+https://github.com/rust-lang/crates.io-index" 363 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 364 | 365 | [[package]] 366 | name = "foldhash" 367 | version = "0.1.4" 368 | source = "registry+https://github.com/rust-lang/crates.io-index" 369 | checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" 370 | 371 | [[package]] 372 | name = "glob" 373 | version = "0.3.2" 374 | source = "registry+https://github.com/rust-lang/crates.io-index" 375 | checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" 376 | 377 | [[package]] 378 | name = "hashbrown" 379 | version = "0.15.2" 380 | source = "registry+https://github.com/rust-lang/crates.io-index" 381 | checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" 382 | dependencies = [ 383 | "foldhash", 384 | ] 385 | 386 | [[package]] 387 | name = "hashlink" 388 | version = "0.10.0" 389 | source = "registry+https://github.com/rust-lang/crates.io-index" 390 | checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" 391 | dependencies = [ 392 | "hashbrown", 393 | ] 394 | 395 | [[package]] 396 | name = "heck" 397 | version = "0.5.0" 398 | source = "registry+https://github.com/rust-lang/crates.io-index" 399 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" 400 | 401 | [[package]] 402 | name = "hex" 403 | version = "0.4.3" 404 | source = "registry+https://github.com/rust-lang/crates.io-index" 405 | checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" 406 | 407 | [[package]] 408 | name = "hex-conservative" 409 | version = "0.2.1" 410 | source = "registry+https://github.com/rust-lang/crates.io-index" 411 | checksum = "5313b072ce3c597065a808dbf612c4c8e8590bdbf8b579508bf7a762c5eae6cd" 412 | dependencies = [ 413 | "arrayvec", 414 | ] 415 | 416 | [[package]] 417 | name = "hex_lit" 418 | version = "0.1.1" 419 | source = "registry+https://github.com/rust-lang/crates.io-index" 420 | checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" 421 | 422 | [[package]] 423 | name = "http" 424 | version = "1.2.0" 425 | source = "registry+https://github.com/rust-lang/crates.io-index" 426 | checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" 427 | dependencies = [ 428 | "bytes", 429 | "fnv", 430 | "itoa", 431 | ] 432 | 433 | [[package]] 434 | name = "httparse" 435 | version = "1.10.1" 436 | source = "registry+https://github.com/rust-lang/crates.io-index" 437 | checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" 438 | 439 | [[package]] 440 | name = "humantime" 441 | version = "2.1.0" 442 | source = "registry+https://github.com/rust-lang/crates.io-index" 443 | checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" 444 | 445 | [[package]] 446 | name = "is_terminal_polyfill" 447 | version = "1.70.1" 448 | source = "registry+https://github.com/rust-lang/crates.io-index" 449 | checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" 450 | 451 | [[package]] 452 | name = "itertools" 453 | version = "0.12.1" 454 | source = "registry+https://github.com/rust-lang/crates.io-index" 455 | checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" 456 | dependencies = [ 457 | "either", 458 | ] 459 | 460 | [[package]] 461 | name = "itoa" 462 | version = "1.0.15" 463 | source = "registry+https://github.com/rust-lang/crates.io-index" 464 | checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" 465 | 466 | [[package]] 467 | name = "jobserver" 468 | version = "0.1.32" 469 | source = "registry+https://github.com/rust-lang/crates.io-index" 470 | checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" 471 | dependencies = [ 472 | "libc", 473 | ] 474 | 475 | [[package]] 476 | name = "lazy_static" 477 | version = "1.5.0" 478 | source = "registry+https://github.com/rust-lang/crates.io-index" 479 | checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" 480 | 481 | [[package]] 482 | name = "lazycell" 483 | version = "1.3.0" 484 | source = "registry+https://github.com/rust-lang/crates.io-index" 485 | checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" 486 | 487 | [[package]] 488 | name = "libc" 489 | version = "0.2.170" 490 | source = "registry+https://github.com/rust-lang/crates.io-index" 491 | checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" 492 | 493 | [[package]] 494 | name = "librocksdb-sys" 495 | version = "0.17.1+9.9.3" 496 | source = "registry+https://github.com/rust-lang/crates.io-index" 497 | checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f" 498 | dependencies = [ 499 | "bindgen", 500 | "bzip2-sys", 501 | "cc", 502 | "libc", 503 | "libz-sys", 504 | "zstd-sys", 505 | ] 506 | 507 | [[package]] 508 | name = "libsqlite3-sys" 509 | version = "0.32.0" 510 | source = "registry+https://github.com/rust-lang/crates.io-index" 511 | checksum = "fbb8270bb4060bd76c6e96f20c52d80620f1d82a3470885694e41e0f81ef6fe7" 512 | dependencies = [ 513 | "pkg-config", 514 | "vcpkg", 515 | ] 516 | 517 | [[package]] 518 | name = "libz-sys" 519 | version = "1.1.21" 520 | source = "registry+https://github.com/rust-lang/crates.io-index" 521 | checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" 522 | dependencies = [ 523 | "cc", 524 | "pkg-config", 525 | "vcpkg", 526 | ] 527 | 528 | [[package]] 529 | name = "log" 530 | version = "0.4.26" 531 | source = "registry+https://github.com/rust-lang/crates.io-index" 532 | checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" 533 | 534 | [[package]] 535 | name = "memchr" 536 | version = "2.7.4" 537 | source = "registry+https://github.com/rust-lang/crates.io-index" 538 | checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" 539 | 540 | [[package]] 541 | name = "minimal-lexical" 542 | version = "0.2.1" 543 | source = "registry+https://github.com/rust-lang/crates.io-index" 544 | checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" 545 | 546 | [[package]] 547 | name = "nom" 548 | version = "7.1.3" 549 | source = "registry+https://github.com/rust-lang/crates.io-index" 550 | checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" 551 | dependencies = [ 552 | "memchr", 553 | "minimal-lexical", 554 | ] 555 | 556 | [[package]] 557 | name = "num-traits" 558 | version = "0.2.19" 559 | source = "registry+https://github.com/rust-lang/crates.io-index" 560 | checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" 561 | dependencies = [ 562 | "autocfg", 563 | ] 564 | 565 | [[package]] 566 | name = "once_cell" 567 | version = "1.20.3" 568 | source = "registry+https://github.com/rust-lang/crates.io-index" 569 | checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" 570 | 571 | [[package]] 572 | name = "papergrid" 573 | version = "0.14.0" 574 | source = "registry+https://github.com/rust-lang/crates.io-index" 575 | checksum = "b915f831b85d984193fdc3d3611505871dc139b2534530fa01c1a6a6707b6723" 576 | dependencies = [ 577 | "bytecount", 578 | "fnv", 579 | "unicode-width", 580 | ] 581 | 582 | [[package]] 583 | name = "percent-encoding" 584 | version = "2.3.1" 585 | source = "registry+https://github.com/rust-lang/crates.io-index" 586 | checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" 587 | 588 | [[package]] 589 | name = "pkg-config" 590 | version = "0.3.32" 591 | source = "registry+https://github.com/rust-lang/crates.io-index" 592 | checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" 593 | 594 | [[package]] 595 | name = "proc-macro-error-attr2" 596 | version = "2.0.0" 597 | source = "registry+https://github.com/rust-lang/crates.io-index" 598 | checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" 599 | dependencies = [ 600 | "proc-macro2", 601 | "quote", 602 | ] 603 | 604 | [[package]] 605 | name = "proc-macro-error2" 606 | version = "2.0.1" 607 | source = "registry+https://github.com/rust-lang/crates.io-index" 608 | checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" 609 | dependencies = [ 610 | "proc-macro-error-attr2", 611 | "proc-macro2", 612 | "quote", 613 | "syn", 614 | ] 615 | 616 | [[package]] 617 | name = "proc-macro2" 618 | version = "1.0.94" 619 | source = "registry+https://github.com/rust-lang/crates.io-index" 620 | checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" 621 | dependencies = [ 622 | "unicode-ident", 623 | ] 624 | 625 | [[package]] 626 | name = "quote" 627 | version = "1.0.39" 628 | source = "registry+https://github.com/rust-lang/crates.io-index" 629 | checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801" 630 | dependencies = [ 631 | "proc-macro2", 632 | ] 633 | 634 | [[package]] 635 | name = "regex" 636 | version = "1.11.1" 637 | source = "registry+https://github.com/rust-lang/crates.io-index" 638 | checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" 639 | dependencies = [ 640 | "aho-corasick", 641 | "memchr", 642 | "regex-automata", 643 | "regex-syntax", 644 | ] 645 | 646 | [[package]] 647 | name = "regex-automata" 648 | version = "0.4.9" 649 | source = "registry+https://github.com/rust-lang/crates.io-index" 650 | checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" 651 | dependencies = [ 652 | "aho-corasick", 653 | "memchr", 654 | "regex-syntax", 655 | ] 656 | 657 | [[package]] 658 | name = "regex-syntax" 659 | version = "0.8.5" 660 | source = "registry+https://github.com/rust-lang/crates.io-index" 661 | checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" 662 | 663 | [[package]] 664 | name = "rocksdb" 665 | version = "0.23.0" 666 | source = "registry+https://github.com/rust-lang/crates.io-index" 667 | checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43" 668 | dependencies = [ 669 | "libc", 670 | "librocksdb-sys", 671 | ] 672 | 673 | [[package]] 674 | name = "rusqlite" 675 | version = "0.34.0" 676 | source = "registry+https://github.com/rust-lang/crates.io-index" 677 | checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" 678 | dependencies = [ 679 | "bitflags", 680 | "fallible-iterator", 681 | "fallible-streaming-iterator", 682 | "hashlink", 683 | "libsqlite3-sys", 684 | "smallvec", 685 | ] 686 | 687 | [[package]] 688 | name = "rustc-hash" 689 | version = "1.1.0" 690 | source = "registry+https://github.com/rust-lang/crates.io-index" 691 | checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" 692 | 693 | [[package]] 694 | name = "secp256k1" 695 | version = "0.29.1" 696 | source = "registry+https://github.com/rust-lang/crates.io-index" 697 | checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" 698 | dependencies = [ 699 | "bitcoin_hashes", 700 | "secp256k1-sys", 701 | ] 702 | 703 | [[package]] 704 | name = "secp256k1-sys" 705 | version = "0.10.1" 706 | source = "registry+https://github.com/rust-lang/crates.io-index" 707 | checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9" 708 | dependencies = [ 709 | "cc", 710 | ] 711 | 712 | [[package]] 713 | name = "shlex" 714 | version = "1.3.0" 715 | source = "registry+https://github.com/rust-lang/crates.io-index" 716 | checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" 717 | 718 | [[package]] 719 | name = "smallvec" 720 | version = "1.14.0" 721 | source = "registry+https://github.com/rust-lang/crates.io-index" 722 | checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" 723 | 724 | [[package]] 725 | name = "strsim" 726 | version = "0.11.1" 727 | source = "registry+https://github.com/rust-lang/crates.io-index" 728 | checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" 729 | 730 | [[package]] 731 | name = "syn" 732 | version = "2.0.99" 733 | source = "registry+https://github.com/rust-lang/crates.io-index" 734 | checksum = "e02e925281e18ffd9d640e234264753c43edc62d64b2d4cf898f1bc5e75f3fc2" 735 | dependencies = [ 736 | "proc-macro2", 737 | "quote", 738 | "unicode-ident", 739 | ] 740 | 741 | [[package]] 742 | name = "tabled" 743 | version = "0.18.0" 744 | source = "registry+https://github.com/rust-lang/crates.io-index" 745 | checksum = "121d8171ee5687a4978d1b244f7d99c43e7385a272185a2f1e1fa4dc0979d444" 746 | dependencies = [ 747 | "papergrid", 748 | "tabled_derive", 749 | ] 750 | 751 | [[package]] 752 | name = "tabled_derive" 753 | version = "0.10.0" 754 | source = "registry+https://github.com/rust-lang/crates.io-index" 755 | checksum = "52d9946811baad81710ec921809e2af67ad77719418673b2a3794932d57b7538" 756 | dependencies = [ 757 | "heck", 758 | "proc-macro-error2", 759 | "proc-macro2", 760 | "quote", 761 | "syn", 762 | ] 763 | 764 | [[package]] 765 | name = "thiserror" 766 | version = "2.0.12" 767 | source = "registry+https://github.com/rust-lang/crates.io-index" 768 | checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" 769 | dependencies = [ 770 | "thiserror-impl", 771 | ] 772 | 773 | [[package]] 774 | name = "thiserror-impl" 775 | version = "2.0.12" 776 | source = "registry+https://github.com/rust-lang/crates.io-index" 777 | checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" 778 | dependencies = [ 779 | "proc-macro2", 780 | "quote", 781 | "syn", 782 | ] 783 | 784 | [[package]] 785 | name = "unicode-ident" 786 | version = "1.0.18" 787 | source = "registry+https://github.com/rust-lang/crates.io-index" 788 | checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" 789 | 790 | [[package]] 791 | name = "unicode-width" 792 | version = "0.2.0" 793 | source = "registry+https://github.com/rust-lang/crates.io-index" 794 | checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" 795 | 796 | [[package]] 797 | name = "ureq" 798 | version = "3.0.8" 799 | source = "registry+https://github.com/rust-lang/crates.io-index" 800 | checksum = "06f78313c985f2fba11100dd06d60dd402d0cabb458af4d94791b8e09c025323" 801 | dependencies = [ 802 | "base64", 803 | "log", 804 | "percent-encoding", 805 | "ureq-proto", 806 | "utf-8", 807 | ] 808 | 809 | [[package]] 810 | name = "ureq-proto" 811 | version = "0.3.3" 812 | source = "registry+https://github.com/rust-lang/crates.io-index" 813 | checksum = "64adb55464bad1ab1aa9229133d0d59d2f679180f4d15f0d9debe616f541f25e" 814 | dependencies = [ 815 | "base64", 816 | "http", 817 | "httparse", 818 | "log", 819 | ] 820 | 821 | [[package]] 822 | name = "utf-8" 823 | version = "0.7.6" 824 | source = "registry+https://github.com/rust-lang/crates.io-index" 825 | checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" 826 | 827 | [[package]] 828 | name = "utf8parse" 829 | version = "0.2.2" 830 | source = "registry+https://github.com/rust-lang/crates.io-index" 831 | checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" 832 | 833 | [[package]] 834 | name = "vcpkg" 835 | version = "0.2.15" 836 | source = "registry+https://github.com/rust-lang/crates.io-index" 837 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 838 | 839 | [[package]] 840 | name = "windows-sys" 841 | version = "0.59.0" 842 | source = "registry+https://github.com/rust-lang/crates.io-index" 843 | checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" 844 | dependencies = [ 845 | "windows-targets", 846 | ] 847 | 848 | [[package]] 849 | name = "windows-targets" 850 | version = "0.52.6" 851 | source = "registry+https://github.com/rust-lang/crates.io-index" 852 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 853 | dependencies = [ 854 | "windows_aarch64_gnullvm", 855 | "windows_aarch64_msvc", 856 | "windows_i686_gnu", 857 | "windows_i686_gnullvm", 858 | "windows_i686_msvc", 859 | "windows_x86_64_gnu", 860 | "windows_x86_64_gnullvm", 861 | "windows_x86_64_msvc", 862 | ] 863 | 864 | [[package]] 865 | name = "windows_aarch64_gnullvm" 866 | version = "0.52.6" 867 | source = "registry+https://github.com/rust-lang/crates.io-index" 868 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 869 | 870 | [[package]] 871 | name = "windows_aarch64_msvc" 872 | version = "0.52.6" 873 | source = "registry+https://github.com/rust-lang/crates.io-index" 874 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 875 | 876 | [[package]] 877 | name = "windows_i686_gnu" 878 | version = "0.52.6" 879 | source = "registry+https://github.com/rust-lang/crates.io-index" 880 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 881 | 882 | [[package]] 883 | name = "windows_i686_gnullvm" 884 | version = "0.52.6" 885 | source = "registry+https://github.com/rust-lang/crates.io-index" 886 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 887 | 888 | [[package]] 889 | name = "windows_i686_msvc" 890 | version = "0.52.6" 891 | source = "registry+https://github.com/rust-lang/crates.io-index" 892 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 893 | 894 | [[package]] 895 | name = "windows_x86_64_gnu" 896 | version = "0.52.6" 897 | source = "registry+https://github.com/rust-lang/crates.io-index" 898 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 899 | 900 | [[package]] 901 | name = "windows_x86_64_gnullvm" 902 | version = "0.52.6" 903 | source = "registry+https://github.com/rust-lang/crates.io-index" 904 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 905 | 906 | [[package]] 907 | name = "windows_x86_64_msvc" 908 | version = "0.52.6" 909 | source = "registry+https://github.com/rust-lang/crates.io-index" 910 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 911 | 912 | [[package]] 913 | name = "zstd-sys" 914 | version = "2.0.14+zstd.1.5.7" 915 | source = "registry+https://github.com/rust-lang/crates.io-index" 916 | checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5" 917 | dependencies = [ 918 | "cc", 919 | "pkg-config", 920 | ] 921 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = [ 4 | "bindex-cli", 5 | "bindex-lib", 6 | ] 7 | -------------------------------------------------------------------------------- /Dockerfile.ci: -------------------------------------------------------------------------------- 1 | FROM debian:trixie AS base 2 | 3 | # Prepare base image 4 | ENV DEBIAN_FRONTEND=noninteractive 5 | RUN apt-get -qqy update 6 | RUN apt-get -qqy install librocksdb-dev libsqlite3-dev 7 | 8 | # Prepare builder image 9 | FROM base AS builder 10 | RUN apt-get -qqy install cargo libclang-dev 11 | 12 | WORKDIR /build/ 13 | COPY . . 14 | 15 | # Build with dynamically linked RocksDB library 16 | ENV ROCKSDB_INCLUDE_DIR=/usr/include 17 | ENV ROCKSDB_LIB_DIR=/usr/lib 18 | RUN cargo build --release --locked --all 19 | 20 | # Copy the binaries into runner image 21 | FROM base AS runner 22 | COPY --from=builder /build/target/release/bindex-cli /usr/local/bin/bindex-cli 23 | 24 | WORKDIR / 25 | 26 | # Sanity check 27 | RUN bindex-cli --version 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) Roman Zeyde. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bitcoin indexing library in Rust 2 | 3 | [![CI](https://github.com/romanz/bindex-rs/actions/workflows/ci.yml/badge.svg)](https://github.com/romanz/bindex-rs/actions) 4 | [![crates.io](https://img.shields.io/crates/v/bindex.svg)](https://crates.io/crates/bindex) 5 | 6 | See [slides](https://docs.google.com/presentation/d/1Zez-6DApKRu59kke4i_g9jwxQlaFKKRpOPdYFYsFXfA/) for more details. 7 | 8 | ## Requirements 9 | 10 | Currently, building an address index and querying it requires the following Bitcoin Core PRs: 11 | 12 | - https://github.com/bitcoin/bitcoin/pull/32540 13 | - https://github.com/bitcoin/bitcoin/pull/32541 14 | 15 | ## Usage 16 | 17 | [![asciicast](https://asciinema.org/a/yFjcbagORZNMtoOPikw0kUlC9.svg)](https://asciinema.org/a/yFjcbagORZNMtoOPikw0kUlC9) 18 | -------------------------------------------------------------------------------- /bindex-cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bindex-cli" 3 | version = "0.0.11" 4 | edition = "2021" 5 | authors = ["Roman Zeyde "] 6 | description = "Bitcoin indexing CLI in Rust" 7 | license = "MIT" 8 | homepage = "https://github.com/romanz/bindex-rs" 9 | repository = "https://github.com/romanz/bindex-rs" 10 | keywords = ["bitcoin", "index", "database", "cli"] 11 | documentation = "https://docs.rs/bindex/" 12 | readme = "../README.md" 13 | 14 | 15 | [dependencies] 16 | bindex = { path = "../bindex-lib" } 17 | chrono = { version = "0.4", default-features = false } 18 | clap = { version = "4", features = ["derive"] } 19 | env_logger = "0.11" 20 | log = "0.4" 21 | rusqlite = "0.34" 22 | tabled = "0.18" 23 | -------------------------------------------------------------------------------- /bindex-cli/src/bin/bindex-cli.rs: -------------------------------------------------------------------------------- 1 | use bindex::{ 2 | address::{self, cache}, 3 | bitcoin::{self, consensus::deserialize, hashes::Hash, BlockHash, Txid}, 4 | cli, 5 | }; 6 | use chrono::{TimeZone, Utc}; 7 | use clap::Parser; 8 | use log::*; 9 | use std::{ 10 | collections::HashSet, 11 | io::{BufRead, BufReader, Read, Write}, 12 | path::{Path, PathBuf}, 13 | process::{ChildStdin, ChildStdout, Command, Stdio}, 14 | str::FromStr, 15 | thread, 16 | time::Instant, 17 | }; 18 | 19 | type Result = std::result::Result>; 20 | 21 | #[derive(tabled::Tabled)] 22 | struct Entry { 23 | txid: String, 24 | time: String, 25 | height: String, 26 | offset: String, 27 | delta: String, 28 | balance: String, 29 | } 30 | 31 | impl Entry { 32 | fn dots() -> Self { 33 | let s = "..."; 34 | Self { 35 | txid: s.to_owned(), 36 | time: s.to_owned(), 37 | height: s.to_owned(), 38 | offset: s.to_owned(), 39 | delta: s.to_owned(), 40 | balance: s.to_owned(), 41 | } 42 | } 43 | } 44 | 45 | fn get_history(db: &rusqlite::Connection) -> Result> { 46 | let t = Instant::now(); 47 | 48 | let addresses: usize = db.query_row("SELECT count(*) FROM watch", [], |row| { 49 | let sum: Option = row.get(0)?; 50 | Ok(sum.unwrap_or(0)) 51 | })?; 52 | if addresses == 0 { 53 | return Ok(vec![]); 54 | } 55 | 56 | let mut select = db.prepare( 57 | r" 58 | WITH history_deltas AS ( 59 | SELECT 60 | block_offset, 61 | block_height, 62 | sum(history.amount) AS delta 63 | FROM history 64 | GROUP BY 1, 2 65 | ) 66 | SELECT 67 | h.header_bytes, 68 | t.block_offset, 69 | t.block_height, 70 | t.tx_id, 71 | d.delta 72 | FROM 73 | history_deltas d, transactions t, headers h 74 | WHERE 75 | d.block_height = t.block_height AND 76 | d.block_offset = t.block_offset AND 77 | d.block_height = h.block_height 78 | ORDER BY 79 | d.block_height ASC, 80 | d.block_offset ASC", 81 | )?; 82 | 83 | let mut balance = bitcoin::SignedAmount::ZERO; 84 | let entries = select 85 | .query([])? 86 | .and_then(|row| -> Result> { 87 | let header_bytes: Vec = row.get(0)?; 88 | let block_offset: u64 = row.get(1)?; 89 | let block_height: usize = row.get(2)?; 90 | let txid = Txid::from_byte_array(row.get(3)?); 91 | let delta = bitcoin::SignedAmount::from_sat(row.get(4)?); 92 | balance += delta; 93 | let header: bitcoin::block::Header = 94 | deserialize(&header_bytes).expect("bad header bytes"); 95 | Ok(Some(Entry { 96 | txid: txid.to_string(), 97 | time: format!("{}", Utc.timestamp_opt(header.time.into(), 0).unwrap()), 98 | height: block_height.to_string(), 99 | offset: block_offset.to_string(), 100 | delta: format!("{:+.8}", delta.to_btc()), 101 | balance: format!("{:.8}", balance.to_btc()), 102 | })) 103 | }) 104 | .filter_map(Result::transpose) 105 | .collect::>>()?; 106 | 107 | assert!(!balance.is_negative()); 108 | let balance_check = db.query_row("SELECT sum(amount) FROM history", [], |row| { 109 | let sum: Option = row.get(0)?; 110 | Ok(bitcoin::SignedAmount::from_sat(sum.unwrap_or(0))) 111 | })?; 112 | assert_eq!(balance_check, balance); 113 | 114 | let utxos: usize = db.query_row("SELECT sum(sign(amount)) FROM history", [], |row| { 115 | let sum: Option = row.get(0)?; 116 | Ok(sum.unwrap_or(0)) 117 | })?; 118 | 119 | info!( 120 | "{} address history: {} entries, balance: {}, UTXOs: {} [{:?}]", 121 | addresses, 122 | entries.len(), 123 | balance, 124 | utxos, 125 | t.elapsed(), 126 | ); 127 | Ok(entries) 128 | } 129 | 130 | fn print_history(mut entries: Vec, history_limit: usize) { 131 | if history_limit > 0 { 132 | let is_truncated = entries.len() > history_limit; 133 | entries.reverse(); 134 | entries.truncate(history_limit); 135 | if is_truncated { 136 | entries.push(Entry::dots()); 137 | } 138 | if entries.is_empty() { 139 | return; 140 | } 141 | let mut tbl = tabled::Table::new(entries); 142 | tbl.with(tabled::settings::Style::rounded()); 143 | tbl.modify( 144 | tabled::settings::object::Rows::new(1..), 145 | tabled::settings::Alignment::right(), 146 | ); 147 | if is_truncated { 148 | tbl.modify( 149 | tabled::settings::object::LastRow, 150 | tabled::settings::Alignment::center(), 151 | ); 152 | } 153 | println!("{}", tbl); 154 | } 155 | } 156 | 157 | #[derive(Parser)] 158 | #[command(version, about, long_about = None)] 159 | /// Bitcoin address indexer 160 | struct Args { 161 | #[arg(value_enum, short = 'n', long = "network", default_value_t = cli::Network::Bitcoin)] 162 | network: cli::Network, 163 | 164 | /// Limit on how many recent transactions to print 165 | #[arg(short = 'l', long = "limit", default_value_t = 10)] 166 | history_limit: usize, 167 | 168 | /// Text file, containing white-space separated addresses to add 169 | #[arg(short = 'a', long = "address-file")] 170 | address_file: Option, 171 | 172 | /// SQLite3 database for storing address history and relevant transactions 173 | #[arg(short = 'c', long = "cache")] 174 | cache_file: Option, 175 | 176 | /// Exit after one sync is over 177 | #[arg(short = '1', long = "sync-once", default_value_t = false)] 178 | sync_once: bool, 179 | 180 | /// Start Electrum server 181 | #[arg(short = 'e', long = "electrum", default_value_t = false)] 182 | electrum: bool, 183 | } 184 | 185 | fn collect_addresses(args: &Args) -> Result> { 186 | let text = args.address_file.as_ref().map_or_else( 187 | || Ok(String::new()), 188 | |path| { 189 | if path == Path::new("-") { 190 | let mut buf = String::new(); 191 | std::io::stdin().read_to_string(&mut buf)?; 192 | return Ok(buf); 193 | } 194 | std::fs::read_to_string(path) 195 | }, 196 | )?; 197 | let addresses = text 198 | .split_whitespace() 199 | .map(|addr| -> Result { 200 | Ok(bitcoin::Address::from_str(addr)?.require_network(args.network.into())?) 201 | }) 202 | .collect::>()?; 203 | Ok(addresses) 204 | } 205 | 206 | struct Electrum { 207 | stdin: ChildStdin, 208 | stdout: BufReader, 209 | line: String, 210 | } 211 | 212 | impl Electrum { 213 | fn start(cache_file: &Path) -> Result { 214 | let mut server = Command::new("python") 215 | .arg("-m") 216 | .arg("electrum.server") 217 | .arg(cache_file) 218 | .stdin(Stdio::piped()) 219 | .stdout(Stdio::piped()) 220 | .spawn()?; 221 | info!("Launched server @ pid={}", server.id()); 222 | 223 | let stdin = server.stdin.take().unwrap(); 224 | let stdout = BufReader::new(server.stdout.take().unwrap()); 225 | 226 | Ok(Self { 227 | stdin, 228 | stdout, 229 | line: String::new(), 230 | }) 231 | } 232 | 233 | fn notify(&mut self, new_tip: BlockHash) -> Result<()> { 234 | debug!("chain best block={}", new_tip); 235 | writeln!(self.stdin, "{}", new_tip)?; 236 | self.stdin.flush()?; 237 | Ok(()) 238 | } 239 | 240 | fn wait(&mut self) -> Result<()> { 241 | self.line.clear(); 242 | self.stdout.read_line(&mut self.line)?; // wait for notification 243 | Ok(()) 244 | } 245 | } 246 | 247 | fn main() -> Result<()> { 248 | let args = Args::parse(); 249 | env_logger::builder().format_timestamp_micros().init(); 250 | let cache_db = rusqlite::Connection::open(Path::new(match args.cache_file { 251 | Some(ref p) => p.as_path(), 252 | None => Path::new(":memory:"), 253 | }))?; 254 | 255 | let cache = cache::Cache::open(cache_db)?; 256 | cache.add(collect_addresses(&args)?)?; 257 | 258 | let mut server = None; 259 | if args.electrum { 260 | let cache_file = args 261 | .cache_file 262 | .ok_or("Electrum requires setting a cache file")?; 263 | server = Some(Electrum::start(&cache_file)?); 264 | } 265 | let mut index = address::Index::open_default(args.network)?; 266 | let mut tip = BlockHash::all_zeros(); 267 | loop { 268 | while index.sync_chain(1000)?.indexed_blocks > 0 {} 269 | if cache.sync(&index, &mut tip)? { 270 | let entries = get_history(cache.db())?; 271 | print_history(entries, args.history_limit); 272 | } 273 | if args.sync_once { 274 | break; 275 | } 276 | match server.as_mut() { 277 | Some(s) => { 278 | s.notify(tip)?; 279 | s.wait()?; // Electrum should send an ACK for an index sync request 280 | } 281 | None => thread::sleep(std::time::Duration::from_secs(1)), 282 | } 283 | } 284 | Ok(()) 285 | } 286 | -------------------------------------------------------------------------------- /bindex-lib/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 4 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "1.1.3" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "anstream" 16 | version = "0.6.18" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" 19 | dependencies = [ 20 | "anstyle", 21 | "anstyle-parse", 22 | "anstyle-query", 23 | "anstyle-wincon", 24 | "colorchoice", 25 | "is_terminal_polyfill", 26 | "utf8parse", 27 | ] 28 | 29 | [[package]] 30 | name = "anstyle" 31 | version = "1.0.10" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" 34 | 35 | [[package]] 36 | name = "anstyle-parse" 37 | version = "0.2.6" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" 40 | dependencies = [ 41 | "utf8parse", 42 | ] 43 | 44 | [[package]] 45 | name = "anstyle-query" 46 | version = "1.1.2" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" 49 | dependencies = [ 50 | "windows-sys", 51 | ] 52 | 53 | [[package]] 54 | name = "anstyle-wincon" 55 | version = "3.0.7" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" 58 | dependencies = [ 59 | "anstyle", 60 | "once_cell", 61 | "windows-sys", 62 | ] 63 | 64 | [[package]] 65 | name = "arrayvec" 66 | version = "0.7.6" 67 | source = "registry+https://github.com/rust-lang/crates.io-index" 68 | checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" 69 | 70 | [[package]] 71 | name = "autocfg" 72 | version = "1.4.0" 73 | source = "registry+https://github.com/rust-lang/crates.io-index" 74 | checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" 75 | 76 | [[package]] 77 | name = "base58ck" 78 | version = "0.1.0" 79 | source = "registry+https://github.com/rust-lang/crates.io-index" 80 | checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f" 81 | dependencies = [ 82 | "bitcoin-internals", 83 | "bitcoin_hashes", 84 | ] 85 | 86 | [[package]] 87 | name = "base64" 88 | version = "0.22.1" 89 | source = "registry+https://github.com/rust-lang/crates.io-index" 90 | checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" 91 | 92 | [[package]] 93 | name = "bech32" 94 | version = "0.11.0" 95 | source = "registry+https://github.com/rust-lang/crates.io-index" 96 | checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" 97 | 98 | [[package]] 99 | name = "bindex" 100 | version = "0.0.6" 101 | dependencies = [ 102 | "bitcoin", 103 | "bitcoin_slices", 104 | "chrono", 105 | "clap", 106 | "env_logger", 107 | "hex", 108 | "hex_lit", 109 | "log", 110 | "rocksdb", 111 | "rusqlite", 112 | "tabled", 113 | "thiserror", 114 | "ureq", 115 | ] 116 | 117 | [[package]] 118 | name = "bindgen" 119 | version = "0.69.5" 120 | source = "registry+https://github.com/rust-lang/crates.io-index" 121 | checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" 122 | dependencies = [ 123 | "bitflags", 124 | "cexpr", 125 | "clang-sys", 126 | "itertools", 127 | "lazy_static", 128 | "lazycell", 129 | "proc-macro2", 130 | "quote", 131 | "regex", 132 | "rustc-hash", 133 | "shlex", 134 | "syn", 135 | ] 136 | 137 | [[package]] 138 | name = "bitcoin" 139 | version = "0.32.5" 140 | source = "registry+https://github.com/rust-lang/crates.io-index" 141 | checksum = "ce6bc65742dea50536e35ad42492b234c27904a27f0abdcbce605015cb4ea026" 142 | dependencies = [ 143 | "base58ck", 144 | "bech32", 145 | "bitcoin-internals", 146 | "bitcoin-io", 147 | "bitcoin-units", 148 | "bitcoin_hashes", 149 | "hex-conservative", 150 | "hex_lit", 151 | "secp256k1", 152 | ] 153 | 154 | [[package]] 155 | name = "bitcoin-internals" 156 | version = "0.3.0" 157 | source = "registry+https://github.com/rust-lang/crates.io-index" 158 | checksum = "30bdbe14aa07b06e6cfeffc529a1f099e5fbe249524f8125358604df99a4bed2" 159 | 160 | [[package]] 161 | name = "bitcoin-io" 162 | version = "0.1.3" 163 | source = "registry+https://github.com/rust-lang/crates.io-index" 164 | checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf" 165 | 166 | [[package]] 167 | name = "bitcoin-units" 168 | version = "0.1.2" 169 | source = "registry+https://github.com/rust-lang/crates.io-index" 170 | checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2" 171 | dependencies = [ 172 | "bitcoin-internals", 173 | ] 174 | 175 | [[package]] 176 | name = "bitcoin_hashes" 177 | version = "0.14.0" 178 | source = "registry+https://github.com/rust-lang/crates.io-index" 179 | checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16" 180 | dependencies = [ 181 | "bitcoin-io", 182 | "hex-conservative", 183 | ] 184 | 185 | [[package]] 186 | name = "bitcoin_slices" 187 | version = "0.10.0" 188 | source = "registry+https://github.com/rust-lang/crates.io-index" 189 | checksum = "7943d4257fdfc85afe2a13d2b539a5213e97bb978329dd79b624acbce73042fb" 190 | dependencies = [ 191 | "bitcoin", 192 | "bitcoin_hashes", 193 | ] 194 | 195 | [[package]] 196 | name = "bitflags" 197 | version = "2.9.0" 198 | source = "registry+https://github.com/rust-lang/crates.io-index" 199 | checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" 200 | 201 | [[package]] 202 | name = "bytecount" 203 | version = "0.6.8" 204 | source = "registry+https://github.com/rust-lang/crates.io-index" 205 | checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" 206 | 207 | [[package]] 208 | name = "bytes" 209 | version = "1.10.1" 210 | source = "registry+https://github.com/rust-lang/crates.io-index" 211 | checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" 212 | 213 | [[package]] 214 | name = "bzip2-sys" 215 | version = "0.1.13+1.0.8" 216 | source = "registry+https://github.com/rust-lang/crates.io-index" 217 | checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" 218 | dependencies = [ 219 | "cc", 220 | "pkg-config", 221 | ] 222 | 223 | [[package]] 224 | name = "cc" 225 | version = "1.2.16" 226 | source = "registry+https://github.com/rust-lang/crates.io-index" 227 | checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" 228 | dependencies = [ 229 | "jobserver", 230 | "libc", 231 | "shlex", 232 | ] 233 | 234 | [[package]] 235 | name = "cexpr" 236 | version = "0.6.0" 237 | source = "registry+https://github.com/rust-lang/crates.io-index" 238 | checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" 239 | dependencies = [ 240 | "nom", 241 | ] 242 | 243 | [[package]] 244 | name = "chrono" 245 | version = "0.4.40" 246 | source = "registry+https://github.com/rust-lang/crates.io-index" 247 | checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" 248 | dependencies = [ 249 | "num-traits", 250 | ] 251 | 252 | [[package]] 253 | name = "clang-sys" 254 | version = "1.8.1" 255 | source = "registry+https://github.com/rust-lang/crates.io-index" 256 | checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" 257 | dependencies = [ 258 | "glob", 259 | "libc", 260 | ] 261 | 262 | [[package]] 263 | name = "clap" 264 | version = "4.5.31" 265 | source = "registry+https://github.com/rust-lang/crates.io-index" 266 | checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" 267 | dependencies = [ 268 | "clap_builder", 269 | "clap_derive", 270 | ] 271 | 272 | [[package]] 273 | name = "clap_builder" 274 | version = "4.5.31" 275 | source = "registry+https://github.com/rust-lang/crates.io-index" 276 | checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" 277 | dependencies = [ 278 | "anstream", 279 | "anstyle", 280 | "clap_lex", 281 | "strsim", 282 | ] 283 | 284 | [[package]] 285 | name = "clap_derive" 286 | version = "4.5.28" 287 | source = "registry+https://github.com/rust-lang/crates.io-index" 288 | checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" 289 | dependencies = [ 290 | "heck", 291 | "proc-macro2", 292 | "quote", 293 | "syn", 294 | ] 295 | 296 | [[package]] 297 | name = "clap_lex" 298 | version = "0.7.4" 299 | source = "registry+https://github.com/rust-lang/crates.io-index" 300 | checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" 301 | 302 | [[package]] 303 | name = "colorchoice" 304 | version = "1.0.3" 305 | source = "registry+https://github.com/rust-lang/crates.io-index" 306 | checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" 307 | 308 | [[package]] 309 | name = "either" 310 | version = "1.15.0" 311 | source = "registry+https://github.com/rust-lang/crates.io-index" 312 | checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" 313 | 314 | [[package]] 315 | name = "env_filter" 316 | version = "0.1.3" 317 | source = "registry+https://github.com/rust-lang/crates.io-index" 318 | checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" 319 | dependencies = [ 320 | "log", 321 | "regex", 322 | ] 323 | 324 | [[package]] 325 | name = "env_logger" 326 | version = "0.11.6" 327 | source = "registry+https://github.com/rust-lang/crates.io-index" 328 | checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" 329 | dependencies = [ 330 | "anstream", 331 | "anstyle", 332 | "env_filter", 333 | "humantime", 334 | "log", 335 | ] 336 | 337 | [[package]] 338 | name = "fallible-iterator" 339 | version = "0.3.0" 340 | source = "registry+https://github.com/rust-lang/crates.io-index" 341 | checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" 342 | 343 | [[package]] 344 | name = "fallible-streaming-iterator" 345 | version = "0.1.9" 346 | source = "registry+https://github.com/rust-lang/crates.io-index" 347 | checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" 348 | 349 | [[package]] 350 | name = "fnv" 351 | version = "1.0.7" 352 | source = "registry+https://github.com/rust-lang/crates.io-index" 353 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 354 | 355 | [[package]] 356 | name = "foldhash" 357 | version = "0.1.4" 358 | source = "registry+https://github.com/rust-lang/crates.io-index" 359 | checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" 360 | 361 | [[package]] 362 | name = "glob" 363 | version = "0.3.2" 364 | source = "registry+https://github.com/rust-lang/crates.io-index" 365 | checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" 366 | 367 | [[package]] 368 | name = "hashbrown" 369 | version = "0.15.2" 370 | source = "registry+https://github.com/rust-lang/crates.io-index" 371 | checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" 372 | dependencies = [ 373 | "foldhash", 374 | ] 375 | 376 | [[package]] 377 | name = "hashlink" 378 | version = "0.10.0" 379 | source = "registry+https://github.com/rust-lang/crates.io-index" 380 | checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" 381 | dependencies = [ 382 | "hashbrown", 383 | ] 384 | 385 | [[package]] 386 | name = "heck" 387 | version = "0.5.0" 388 | source = "registry+https://github.com/rust-lang/crates.io-index" 389 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" 390 | 391 | [[package]] 392 | name = "hex" 393 | version = "0.4.3" 394 | source = "registry+https://github.com/rust-lang/crates.io-index" 395 | checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" 396 | 397 | [[package]] 398 | name = "hex-conservative" 399 | version = "0.2.1" 400 | source = "registry+https://github.com/rust-lang/crates.io-index" 401 | checksum = "5313b072ce3c597065a808dbf612c4c8e8590bdbf8b579508bf7a762c5eae6cd" 402 | dependencies = [ 403 | "arrayvec", 404 | ] 405 | 406 | [[package]] 407 | name = "hex_lit" 408 | version = "0.1.1" 409 | source = "registry+https://github.com/rust-lang/crates.io-index" 410 | checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" 411 | 412 | [[package]] 413 | name = "http" 414 | version = "1.2.0" 415 | source = "registry+https://github.com/rust-lang/crates.io-index" 416 | checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" 417 | dependencies = [ 418 | "bytes", 419 | "fnv", 420 | "itoa", 421 | ] 422 | 423 | [[package]] 424 | name = "httparse" 425 | version = "1.10.1" 426 | source = "registry+https://github.com/rust-lang/crates.io-index" 427 | checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" 428 | 429 | [[package]] 430 | name = "humantime" 431 | version = "2.1.0" 432 | source = "registry+https://github.com/rust-lang/crates.io-index" 433 | checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" 434 | 435 | [[package]] 436 | name = "is_terminal_polyfill" 437 | version = "1.70.1" 438 | source = "registry+https://github.com/rust-lang/crates.io-index" 439 | checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" 440 | 441 | [[package]] 442 | name = "itertools" 443 | version = "0.12.1" 444 | source = "registry+https://github.com/rust-lang/crates.io-index" 445 | checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" 446 | dependencies = [ 447 | "either", 448 | ] 449 | 450 | [[package]] 451 | name = "itoa" 452 | version = "1.0.15" 453 | source = "registry+https://github.com/rust-lang/crates.io-index" 454 | checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" 455 | 456 | [[package]] 457 | name = "jobserver" 458 | version = "0.1.32" 459 | source = "registry+https://github.com/rust-lang/crates.io-index" 460 | checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" 461 | dependencies = [ 462 | "libc", 463 | ] 464 | 465 | [[package]] 466 | name = "lazy_static" 467 | version = "1.5.0" 468 | source = "registry+https://github.com/rust-lang/crates.io-index" 469 | checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" 470 | 471 | [[package]] 472 | name = "lazycell" 473 | version = "1.3.0" 474 | source = "registry+https://github.com/rust-lang/crates.io-index" 475 | checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" 476 | 477 | [[package]] 478 | name = "libc" 479 | version = "0.2.170" 480 | source = "registry+https://github.com/rust-lang/crates.io-index" 481 | checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" 482 | 483 | [[package]] 484 | name = "librocksdb-sys" 485 | version = "0.17.1+9.9.3" 486 | source = "registry+https://github.com/rust-lang/crates.io-index" 487 | checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f" 488 | dependencies = [ 489 | "bindgen", 490 | "bzip2-sys", 491 | "cc", 492 | "libc", 493 | "libz-sys", 494 | "zstd-sys", 495 | ] 496 | 497 | [[package]] 498 | name = "libsqlite3-sys" 499 | version = "0.32.0" 500 | source = "registry+https://github.com/rust-lang/crates.io-index" 501 | checksum = "fbb8270bb4060bd76c6e96f20c52d80620f1d82a3470885694e41e0f81ef6fe7" 502 | dependencies = [ 503 | "pkg-config", 504 | "vcpkg", 505 | ] 506 | 507 | [[package]] 508 | name = "libz-sys" 509 | version = "1.1.21" 510 | source = "registry+https://github.com/rust-lang/crates.io-index" 511 | checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" 512 | dependencies = [ 513 | "cc", 514 | "pkg-config", 515 | "vcpkg", 516 | ] 517 | 518 | [[package]] 519 | name = "log" 520 | version = "0.4.26" 521 | source = "registry+https://github.com/rust-lang/crates.io-index" 522 | checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" 523 | 524 | [[package]] 525 | name = "memchr" 526 | version = "2.7.4" 527 | source = "registry+https://github.com/rust-lang/crates.io-index" 528 | checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" 529 | 530 | [[package]] 531 | name = "minimal-lexical" 532 | version = "0.2.1" 533 | source = "registry+https://github.com/rust-lang/crates.io-index" 534 | checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" 535 | 536 | [[package]] 537 | name = "nom" 538 | version = "7.1.3" 539 | source = "registry+https://github.com/rust-lang/crates.io-index" 540 | checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" 541 | dependencies = [ 542 | "memchr", 543 | "minimal-lexical", 544 | ] 545 | 546 | [[package]] 547 | name = "num-traits" 548 | version = "0.2.19" 549 | source = "registry+https://github.com/rust-lang/crates.io-index" 550 | checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" 551 | dependencies = [ 552 | "autocfg", 553 | ] 554 | 555 | [[package]] 556 | name = "once_cell" 557 | version = "1.20.3" 558 | source = "registry+https://github.com/rust-lang/crates.io-index" 559 | checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" 560 | 561 | [[package]] 562 | name = "papergrid" 563 | version = "0.14.0" 564 | source = "registry+https://github.com/rust-lang/crates.io-index" 565 | checksum = "b915f831b85d984193fdc3d3611505871dc139b2534530fa01c1a6a6707b6723" 566 | dependencies = [ 567 | "bytecount", 568 | "fnv", 569 | "unicode-width", 570 | ] 571 | 572 | [[package]] 573 | name = "percent-encoding" 574 | version = "2.3.1" 575 | source = "registry+https://github.com/rust-lang/crates.io-index" 576 | checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" 577 | 578 | [[package]] 579 | name = "pkg-config" 580 | version = "0.3.32" 581 | source = "registry+https://github.com/rust-lang/crates.io-index" 582 | checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" 583 | 584 | [[package]] 585 | name = "proc-macro-error-attr2" 586 | version = "2.0.0" 587 | source = "registry+https://github.com/rust-lang/crates.io-index" 588 | checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" 589 | dependencies = [ 590 | "proc-macro2", 591 | "quote", 592 | ] 593 | 594 | [[package]] 595 | name = "proc-macro-error2" 596 | version = "2.0.1" 597 | source = "registry+https://github.com/rust-lang/crates.io-index" 598 | checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" 599 | dependencies = [ 600 | "proc-macro-error-attr2", 601 | "proc-macro2", 602 | "quote", 603 | "syn", 604 | ] 605 | 606 | [[package]] 607 | name = "proc-macro2" 608 | version = "1.0.94" 609 | source = "registry+https://github.com/rust-lang/crates.io-index" 610 | checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" 611 | dependencies = [ 612 | "unicode-ident", 613 | ] 614 | 615 | [[package]] 616 | name = "quote" 617 | version = "1.0.39" 618 | source = "registry+https://github.com/rust-lang/crates.io-index" 619 | checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801" 620 | dependencies = [ 621 | "proc-macro2", 622 | ] 623 | 624 | [[package]] 625 | name = "regex" 626 | version = "1.11.1" 627 | source = "registry+https://github.com/rust-lang/crates.io-index" 628 | checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" 629 | dependencies = [ 630 | "aho-corasick", 631 | "memchr", 632 | "regex-automata", 633 | "regex-syntax", 634 | ] 635 | 636 | [[package]] 637 | name = "regex-automata" 638 | version = "0.4.9" 639 | source = "registry+https://github.com/rust-lang/crates.io-index" 640 | checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" 641 | dependencies = [ 642 | "aho-corasick", 643 | "memchr", 644 | "regex-syntax", 645 | ] 646 | 647 | [[package]] 648 | name = "regex-syntax" 649 | version = "0.8.5" 650 | source = "registry+https://github.com/rust-lang/crates.io-index" 651 | checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" 652 | 653 | [[package]] 654 | name = "rocksdb" 655 | version = "0.23.0" 656 | source = "registry+https://github.com/rust-lang/crates.io-index" 657 | checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43" 658 | dependencies = [ 659 | "libc", 660 | "librocksdb-sys", 661 | ] 662 | 663 | [[package]] 664 | name = "rusqlite" 665 | version = "0.34.0" 666 | source = "registry+https://github.com/rust-lang/crates.io-index" 667 | checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" 668 | dependencies = [ 669 | "bitflags", 670 | "fallible-iterator", 671 | "fallible-streaming-iterator", 672 | "hashlink", 673 | "libsqlite3-sys", 674 | "smallvec", 675 | ] 676 | 677 | [[package]] 678 | name = "rustc-hash" 679 | version = "1.1.0" 680 | source = "registry+https://github.com/rust-lang/crates.io-index" 681 | checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" 682 | 683 | [[package]] 684 | name = "secp256k1" 685 | version = "0.29.1" 686 | source = "registry+https://github.com/rust-lang/crates.io-index" 687 | checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" 688 | dependencies = [ 689 | "bitcoin_hashes", 690 | "secp256k1-sys", 691 | ] 692 | 693 | [[package]] 694 | name = "secp256k1-sys" 695 | version = "0.10.1" 696 | source = "registry+https://github.com/rust-lang/crates.io-index" 697 | checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9" 698 | dependencies = [ 699 | "cc", 700 | ] 701 | 702 | [[package]] 703 | name = "shlex" 704 | version = "1.3.0" 705 | source = "registry+https://github.com/rust-lang/crates.io-index" 706 | checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" 707 | 708 | [[package]] 709 | name = "smallvec" 710 | version = "1.14.0" 711 | source = "registry+https://github.com/rust-lang/crates.io-index" 712 | checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" 713 | 714 | [[package]] 715 | name = "strsim" 716 | version = "0.11.1" 717 | source = "registry+https://github.com/rust-lang/crates.io-index" 718 | checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" 719 | 720 | [[package]] 721 | name = "syn" 722 | version = "2.0.99" 723 | source = "registry+https://github.com/rust-lang/crates.io-index" 724 | checksum = "e02e925281e18ffd9d640e234264753c43edc62d64b2d4cf898f1bc5e75f3fc2" 725 | dependencies = [ 726 | "proc-macro2", 727 | "quote", 728 | "unicode-ident", 729 | ] 730 | 731 | [[package]] 732 | name = "tabled" 733 | version = "0.18.0" 734 | source = "registry+https://github.com/rust-lang/crates.io-index" 735 | checksum = "121d8171ee5687a4978d1b244f7d99c43e7385a272185a2f1e1fa4dc0979d444" 736 | dependencies = [ 737 | "papergrid", 738 | "tabled_derive", 739 | ] 740 | 741 | [[package]] 742 | name = "tabled_derive" 743 | version = "0.10.0" 744 | source = "registry+https://github.com/rust-lang/crates.io-index" 745 | checksum = "52d9946811baad81710ec921809e2af67ad77719418673b2a3794932d57b7538" 746 | dependencies = [ 747 | "heck", 748 | "proc-macro-error2", 749 | "proc-macro2", 750 | "quote", 751 | "syn", 752 | ] 753 | 754 | [[package]] 755 | name = "thiserror" 756 | version = "2.0.12" 757 | source = "registry+https://github.com/rust-lang/crates.io-index" 758 | checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" 759 | dependencies = [ 760 | "thiserror-impl", 761 | ] 762 | 763 | [[package]] 764 | name = "thiserror-impl" 765 | version = "2.0.12" 766 | source = "registry+https://github.com/rust-lang/crates.io-index" 767 | checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" 768 | dependencies = [ 769 | "proc-macro2", 770 | "quote", 771 | "syn", 772 | ] 773 | 774 | [[package]] 775 | name = "unicode-ident" 776 | version = "1.0.18" 777 | source = "registry+https://github.com/rust-lang/crates.io-index" 778 | checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" 779 | 780 | [[package]] 781 | name = "unicode-width" 782 | version = "0.2.0" 783 | source = "registry+https://github.com/rust-lang/crates.io-index" 784 | checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" 785 | 786 | [[package]] 787 | name = "ureq" 788 | version = "3.0.8" 789 | source = "registry+https://github.com/rust-lang/crates.io-index" 790 | checksum = "06f78313c985f2fba11100dd06d60dd402d0cabb458af4d94791b8e09c025323" 791 | dependencies = [ 792 | "base64", 793 | "log", 794 | "percent-encoding", 795 | "ureq-proto", 796 | "utf-8", 797 | ] 798 | 799 | [[package]] 800 | name = "ureq-proto" 801 | version = "0.3.3" 802 | source = "registry+https://github.com/rust-lang/crates.io-index" 803 | checksum = "64adb55464bad1ab1aa9229133d0d59d2f679180f4d15f0d9debe616f541f25e" 804 | dependencies = [ 805 | "base64", 806 | "http", 807 | "httparse", 808 | "log", 809 | ] 810 | 811 | [[package]] 812 | name = "utf-8" 813 | version = "0.7.6" 814 | source = "registry+https://github.com/rust-lang/crates.io-index" 815 | checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" 816 | 817 | [[package]] 818 | name = "utf8parse" 819 | version = "0.2.2" 820 | source = "registry+https://github.com/rust-lang/crates.io-index" 821 | checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" 822 | 823 | [[package]] 824 | name = "vcpkg" 825 | version = "0.2.15" 826 | source = "registry+https://github.com/rust-lang/crates.io-index" 827 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 828 | 829 | [[package]] 830 | name = "windows-sys" 831 | version = "0.59.0" 832 | source = "registry+https://github.com/rust-lang/crates.io-index" 833 | checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" 834 | dependencies = [ 835 | "windows-targets", 836 | ] 837 | 838 | [[package]] 839 | name = "windows-targets" 840 | version = "0.52.6" 841 | source = "registry+https://github.com/rust-lang/crates.io-index" 842 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 843 | dependencies = [ 844 | "windows_aarch64_gnullvm", 845 | "windows_aarch64_msvc", 846 | "windows_i686_gnu", 847 | "windows_i686_gnullvm", 848 | "windows_i686_msvc", 849 | "windows_x86_64_gnu", 850 | "windows_x86_64_gnullvm", 851 | "windows_x86_64_msvc", 852 | ] 853 | 854 | [[package]] 855 | name = "windows_aarch64_gnullvm" 856 | version = "0.52.6" 857 | source = "registry+https://github.com/rust-lang/crates.io-index" 858 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 859 | 860 | [[package]] 861 | name = "windows_aarch64_msvc" 862 | version = "0.52.6" 863 | source = "registry+https://github.com/rust-lang/crates.io-index" 864 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 865 | 866 | [[package]] 867 | name = "windows_i686_gnu" 868 | version = "0.52.6" 869 | source = "registry+https://github.com/rust-lang/crates.io-index" 870 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 871 | 872 | [[package]] 873 | name = "windows_i686_gnullvm" 874 | version = "0.52.6" 875 | source = "registry+https://github.com/rust-lang/crates.io-index" 876 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 877 | 878 | [[package]] 879 | name = "windows_i686_msvc" 880 | version = "0.52.6" 881 | source = "registry+https://github.com/rust-lang/crates.io-index" 882 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 883 | 884 | [[package]] 885 | name = "windows_x86_64_gnu" 886 | version = "0.52.6" 887 | source = "registry+https://github.com/rust-lang/crates.io-index" 888 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 889 | 890 | [[package]] 891 | name = "windows_x86_64_gnullvm" 892 | version = "0.52.6" 893 | source = "registry+https://github.com/rust-lang/crates.io-index" 894 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 895 | 896 | [[package]] 897 | name = "windows_x86_64_msvc" 898 | version = "0.52.6" 899 | source = "registry+https://github.com/rust-lang/crates.io-index" 900 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 901 | 902 | [[package]] 903 | name = "zstd-sys" 904 | version = "2.0.14+zstd.1.5.7" 905 | source = "registry+https://github.com/rust-lang/crates.io-index" 906 | checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5" 907 | dependencies = [ 908 | "cc", 909 | "pkg-config", 910 | ] 911 | -------------------------------------------------------------------------------- /bindex-lib/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bindex" 3 | version = "0.0.11" 4 | edition = "2021" 5 | authors = ["Roman Zeyde "] 6 | description = "Bitcoin indexing library in Rust" 7 | license = "MIT" 8 | homepage = "https://github.com/romanz/bindex-rs" 9 | repository = "https://github.com/romanz/bindex-rs" 10 | keywords = ["bitcoin", "index", "database"] 11 | documentation = "https://docs.rs/bindex/" 12 | readme = "../README.md" 13 | 14 | 15 | [dependencies] 16 | bitcoin = { version = "0.32", default-features = false } 17 | bitcoin_slices = { version = "0.10", features = ["bitcoin"] } 18 | clap = { version = "4", features = ["derive"] } 19 | hex = "0.4" 20 | log = "0.4" 21 | rocksdb = { version = "0.23", default-features = false, features = ["zstd"]} 22 | thiserror = "2.0" 23 | ureq = { version = "3", default-features = false } 24 | rusqlite = "0.34" 25 | 26 | [dev-dependencies] 27 | hex_lit = "0.1" 28 | -------------------------------------------------------------------------------- /bindex-lib/src/address/cache.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::BTreeSet, fmt::Debug}; 2 | 3 | use bitcoin::{ 4 | consensus::{deserialize, serialize}, 5 | hashes::Hash, 6 | BlockHash, Txid, 7 | }; 8 | use bitcoin_slices::{bsl, Parse}; 9 | use log::*; 10 | use rusqlite::OptionalExtension; 11 | 12 | use crate::{ 13 | address, 14 | chain::{self, Chain, Location}, 15 | index::{self, ScriptHash}, 16 | }; 17 | 18 | #[derive(thiserror::Error, Debug)] 19 | pub enum Error { 20 | #[error("rusqlite failed: {0}")] 21 | DB(#[from] rusqlite::Error), 22 | 23 | #[error("address index failed: {0}")] 24 | Index(#[from] address::Error), 25 | 26 | #[error("parse failed: {0}")] 27 | Address(#[from] bitcoin::address::ParseError), 28 | 29 | #[error("block not found: {0}")] 30 | BlockNotFound(#[from] chain::Reorg), 31 | } 32 | 33 | pub struct Cache { 34 | db: rusqlite::Connection, 35 | } 36 | 37 | impl Cache { 38 | pub fn open(db: rusqlite::Connection) -> Result { 39 | // must be explicitly set outside a transaction - otherwise foreign keys constraints are ignored :( 40 | // https://www.sqlite.org/pragma.html#pragma_foreign_keys 41 | db.execute("PRAGMA foreign_keys = ON", ())?; 42 | let c = Cache { db }; 43 | c.run("create", || c.create_tables())?; 44 | Ok(c) 45 | } 46 | 47 | fn run(&self, op: &str, f: impl FnOnce() -> Result) -> Result { 48 | let start = std::time::Instant::now(); 49 | self.db.execute("BEGIN", ())?; 50 | let res = match f() { 51 | Ok(v) => { 52 | self.db.execute("COMMIT", ())?; 53 | Ok(v) 54 | } 55 | Err(e) => { 56 | self.db.execute("ROLLBACK", ())?; 57 | Err(e) 58 | } 59 | }; 60 | debug!("DB {} took {:?}, result={:?}", op, start.elapsed(), res); 61 | res 62 | } 63 | 64 | fn create_tables(&self) -> Result<(), Error> { 65 | self.db.execute( 66 | r" 67 | CREATE TABLE IF NOT EXISTS headers ( 68 | block_height INTEGER NOT NULL, 69 | block_hash BLOB NOT NULL, 70 | header_bytes BLOB NOT NULL, 71 | PRIMARY KEY (block_height), 72 | UNIQUE (block_hash) 73 | ) WITHOUT ROWID", 74 | (), 75 | )?; 76 | self.db.execute( 77 | r" 78 | CREATE TABLE IF NOT EXISTS transactions ( 79 | block_height INTEGER NOT NULL, 80 | block_offset INTEGER NOT NULL, 81 | tx_id BLOB, 82 | tx_bytes BLOB, 83 | PRIMARY KEY (block_height, block_offset), 84 | UNIQUE (tx_id), 85 | FOREIGN KEY (block_height) REFERENCES headers (block_height) ON DELETE CASCADE 86 | ) WITHOUT ROWID", 87 | (), 88 | )?; 89 | self.db.execute( 90 | r" 91 | CREATE TABLE IF NOT EXISTS history ( 92 | script_hash BLOB NOT NULL REFERENCES watch (script_hash) ON DELETE CASCADE, 93 | block_height INTEGER NOT NULL, 94 | block_offset INTEGER NOT NULL, 95 | is_output BOOLEAN NOT NULL, -- is it funding the address or not (= spending from it) 96 | index_ INTEGER NOT NULL, -- input/output index within a transaction 97 | amount INTEGER NOT NULL, -- in Satoshis (positive=funding, negative=spending) 98 | PRIMARY KEY (script_hash, block_height, block_offset, is_output, index_) 99 | FOREIGN KEY (block_height, block_offset) REFERENCES transactions (block_height, block_offset) ON DELETE CASCADE 100 | ) WITHOUT ROWID", 101 | (), 102 | )?; 103 | self.db.execute( 104 | r" 105 | CREATE TABLE IF NOT EXISTS watch ( 106 | script_hash BLOB NOT NULL, 107 | script_bytes BLOB, 108 | address TEXT, 109 | is_active BOOLEAN, -- is this address being actively watched (starts as NULL) 110 | PRIMARY KEY (script_hash) 111 | ) WITHOUT ROWID", 112 | (), 113 | )?; 114 | Ok(()) 115 | } 116 | 117 | pub fn sync(&self, index: &address::Index, tip: &mut BlockHash) -> Result { 118 | self.run("sync", || { 119 | self.drop_stale_blocks(&index.chain)?; 120 | let new_tip = index.chain.tip_hash().unwrap_or_else(BlockHash::all_zeros); 121 | if *tip == new_tip && self.all_active()? { 122 | return Ok(false); 123 | } 124 | let new_history = self.new_history(index)?; 125 | let new_locations: BTreeSet<_> = new_history.iter().map(|(_, loc)| loc).collect(); 126 | let new_headers: BTreeSet<_> = new_locations 127 | .iter() 128 | .map(|&loc| (loc.height, loc.indexed_header)) 129 | .collect(); 130 | if !new_history.is_empty() { 131 | info!( 132 | "adding {} history entries, {} transactions, {} headers to cache={:?}", 133 | new_history.len(), 134 | new_locations.len(), 135 | new_headers.len(), 136 | self.db.path().unwrap_or("") 137 | ); 138 | } 139 | 140 | self.sync_headers(new_headers.into_iter())?; 141 | self.sync_transactions(new_locations.into_iter(), index)?; 142 | self.sync_history(new_history.into_iter())?; 143 | self.sync_watch()?; 144 | *tip = new_tip; 145 | Ok(true) 146 | }) 147 | } 148 | 149 | pub fn all_active(&self) -> Result { 150 | Ok(self.db.query_row( 151 | "SELECT EXISTS (SELECT 1 FROM watch WHERE NOT ifnull(is_active, FALSE))", 152 | [], 153 | |row| { 154 | let has_inactive: bool = row.get(0)?; 155 | Ok(!has_inactive) 156 | }, 157 | )?) 158 | } 159 | 160 | pub fn drop_stale_blocks(&self, chain: &Chain) -> Result<(), Error> { 161 | let mut select = self 162 | .db 163 | .prepare("SELECT block_hash, block_height FROM headers ORDER BY block_height DESC")?; 164 | let rows = select.query_map((), |row| { 165 | let hash = bitcoin::BlockHash::from_byte_array(row.get(0)?); 166 | let height: usize = row.get(1)?; 167 | Ok((hash, height)) 168 | })?; 169 | let mut delete_from = None; 170 | for row in rows { 171 | let (hash, height) = row?; 172 | match chain.get_header(hash, height) { 173 | Ok(_header) => break, 174 | Err(err) => { 175 | warn!("reorg detected: {}", err); 176 | delete_from = Some(height); 177 | } 178 | } 179 | } 180 | if let Some(height) = delete_from { 181 | let mut delete = self 182 | .db 183 | .prepare("DELETE FROM headers WHERE block_height >= ?1")?; 184 | delete.execute([height])?; 185 | } 186 | Ok(()) 187 | } 188 | 189 | fn new_history<'a>( 190 | &self, 191 | index: &'a address::Index, 192 | ) -> Result)>, Error> { 193 | let mut stmt = self.db.prepare("SELECT script_hash FROM watch")?; 194 | let results = stmt.query_map((), |row| Ok(ScriptHash::from_byte_array(row.get(0)?)))?; 195 | 196 | let mut history = BTreeSet::<(ScriptHash, Location<'a>)>::new(); 197 | for res in results { 198 | let script_hash = res?; 199 | self.new_history_for_script_hash(&script_hash, index, &mut history)?; 200 | } 201 | Ok(history) 202 | } 203 | 204 | fn new_history_for_script_hash<'a>( 205 | &self, 206 | script_hash: &ScriptHash, 207 | index: &'a address::Index, 208 | history: &mut BTreeSet<(ScriptHash, Location<'a>)>, 209 | ) -> Result<(), Error> { 210 | let chain = index.chain(); 211 | let from = self 212 | .last_indexed_header(script_hash, chain)? 213 | .map(index::Header::next_txnum) 214 | .unwrap_or_default(); 215 | index.find_locations(script_hash, from)?.for_each(|loc| { 216 | history.insert((*script_hash, loc)); 217 | }); 218 | Ok(()) 219 | } 220 | 221 | fn sync_headers<'a>( 222 | &self, 223 | entries: impl Iterator, 224 | ) -> Result { 225 | let mut insert = self 226 | .db 227 | .prepare("INSERT OR IGNORE INTO headers VALUES (?1, ?2, ?3)")?; 228 | let mut rows = 0; 229 | for (height, header) in entries { 230 | rows += insert.execute(( 231 | height, 232 | header.hash().as_byte_array(), 233 | serialize(header.header()), 234 | ))?; 235 | } 236 | Ok(rows) 237 | } 238 | 239 | fn sync_transactions<'a>( 240 | &self, 241 | locations: impl Iterator>, 242 | index: &address::Index, 243 | ) -> Result { 244 | let mut insert = self 245 | .db 246 | .prepare("INSERT OR IGNORE INTO transactions VALUES (?1, ?2, ?3, ?4)")?; 247 | let mut rows = 0; 248 | for loc in locations { 249 | let tx_bytes = index.get_tx_bytes(loc).expect("missing tx bytes"); 250 | let parsed = bsl::Transaction::parse(&tx_bytes).expect("invalid tx"); 251 | let txid = Txid::from(parsed.parsed().txid()).to_byte_array(); 252 | rows += insert.execute((loc.height, loc.offset, txid, tx_bytes))?; 253 | } 254 | Ok(rows) 255 | } 256 | 257 | fn sync_history<'a>( 258 | &self, 259 | entries: impl Iterator)>, 260 | ) -> Result { 261 | let mut insert = self 262 | .db 263 | .prepare("INSERT INTO history VALUES (?1, ?2, ?3, ?4, ?5, ?6)")?; 264 | let mut rows = 0; 265 | for (script_hash, loc) in entries { 266 | let tx: bitcoin::Transaction = self.db.query_row( 267 | "SELECT tx_bytes FROM transactions WHERE block_height = ?1 AND block_offset = ?2", 268 | (loc.height, loc.offset), 269 | |row| { 270 | let tx_bytes: Vec = row.get(0)?; 271 | Ok(deserialize(&tx_bytes).expect("invalid tx")) 272 | }, 273 | )?; 274 | // Add spending entries 275 | for (i, input) in tx.input.iter().enumerate() { 276 | let prevout = input.previous_output; 277 | // txid -> (height, offset) 278 | let result = self 279 | .db 280 | .query_row( 281 | "SELECT block_height, block_offset FROM transactions WHERE tx_id = ?1", 282 | [prevout.txid.as_byte_array()], 283 | |row| Ok((row.get(0)?, row.get(1)?)), 284 | ) 285 | .optional()?; 286 | let (height, offset): (usize, usize) = match result { 287 | Some(v) => v, 288 | None => continue, 289 | }; 290 | // (script_hash, height, offset, `true`, index) -> amount 291 | let result: Option = self.db.query_row( 292 | "SELECT amount FROM history WHERE script_hash = ?1 AND block_height = ?2 AND block_offset = ?3 AND is_output = TRUE AND index_ = ?4", 293 | (script_hash.as_byte_array(), height, offset, prevout.vout), 294 | |row| row.get(0) 295 | ).optional()?; 296 | // Skip if not found (e.g. an input spending another script_hash) 297 | if let Some(amount) = result { 298 | assert!(amount > 0); 299 | rows += insert.execute(( 300 | script_hash.as_byte_array(), 301 | loc.height, 302 | loc.offset, 303 | false, 304 | i, 305 | -amount, 306 | ))?; 307 | } 308 | } 309 | // Add funding entries 310 | for (i, output) in tx.output.iter().enumerate() { 311 | // Skip if funding another script_hash 312 | if script_hash == ScriptHash::new(&output.script_pubkey) { 313 | rows += insert.execute(( 314 | script_hash.as_byte_array(), 315 | loc.height, 316 | loc.offset, 317 | true, 318 | i, 319 | output.value.to_sat(), 320 | ))?; 321 | } 322 | } 323 | } 324 | Ok(rows) 325 | } 326 | 327 | fn sync_watch(&self) -> Result { 328 | Ok(self 329 | .db 330 | .prepare("UPDATE watch SET is_active = TRUE")? 331 | .execute([])?) 332 | } 333 | fn last_indexed_header<'a>( 334 | &self, 335 | script_hash: &ScriptHash, 336 | chain: &'a Chain, 337 | ) -> Result, Error> { 338 | let mut stmt = self.db.prepare( 339 | r" 340 | SELECT block_hash, block_height 341 | FROM history INNER JOIN headers USING (block_height) 342 | WHERE script_hash = ?1 343 | ORDER BY block_height DESC 344 | LIMIT 1", 345 | )?; 346 | stmt.query_row([script_hash.as_byte_array()], |row| { 347 | let blockhash = bitcoin::BlockHash::from_byte_array(row.get(0)?); 348 | let height: usize = row.get(1)?; 349 | Ok((blockhash, height)) 350 | }) 351 | .optional()? 352 | .map(|(blockhash, height)| Ok(chain.get_header(blockhash, height)?)) 353 | .transpose() 354 | } 355 | 356 | pub fn add(&self, addresses: impl IntoIterator) -> Result<(), Error> { 357 | let mut insert = self 358 | .db 359 | .prepare("INSERT OR IGNORE INTO watch VALUES (?1, ?2, ?3, NULL)")?; 360 | let mut rows = 0; 361 | for addr in addresses { 362 | let script = addr.script_pubkey(); 363 | let script_hash = ScriptHash::new(&script); 364 | rows += insert.execute(( 365 | script_hash.as_byte_array(), 366 | script.as_bytes(), 367 | addr.to_string(), 368 | ))?; 369 | } 370 | if rows > 0 { 371 | info!("added {} new addresses to watch", rows); 372 | } 373 | Ok(()) 374 | } 375 | 376 | pub fn db(&self) -> &rusqlite::Connection { 377 | &self.db 378 | } 379 | } 380 | -------------------------------------------------------------------------------- /bindex-lib/src/address/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod cache; 2 | 3 | use std::{path::Path, time::Duration}; 4 | 5 | use bitcoin::hashes::Hash; 6 | use log::*; 7 | 8 | use crate::{ 9 | chain::{self, Chain, Location}, 10 | cli, client, db, index, 11 | }; 12 | 13 | #[derive(thiserror::Error, Debug)] 14 | pub enum Error { 15 | #[error("client failed: {0}")] 16 | Client(#[from] client::Error), 17 | 18 | #[error("indexing failed: {0:?}")] 19 | Index(#[from] index::Error), 20 | 21 | #[error("DB failed: {0}")] 22 | DB(#[from] rocksdb::Error), 23 | 24 | #[error("Genesis block hash mismatch: {0} != {1}")] 25 | ChainMismatch(bitcoin::BlockHash, bitcoin::BlockHash), 26 | } 27 | 28 | pub struct Index { 29 | genesis_hash: bitcoin::BlockHash, 30 | chain: chain::Chain, 31 | client: client::Client, 32 | store: db::Store, 33 | } 34 | 35 | pub struct Stats { 36 | pub tip: bitcoin::BlockHash, 37 | pub indexed_blocks: usize, 38 | pub size_read: usize, 39 | pub elapsed: std::time::Duration, 40 | } 41 | 42 | impl Stats { 43 | fn new(tip: bitcoin::BlockHash) -> Self { 44 | Self { 45 | tip, 46 | indexed_blocks: 0, 47 | size_read: 0, 48 | elapsed: Duration::ZERO, 49 | } 50 | } 51 | } 52 | 53 | impl Index { 54 | pub fn open(db_path: impl AsRef, url: impl Into) -> Result { 55 | let db_path = db_path.as_ref(); 56 | let url = url.into(); 57 | info!("index DB: {:?}, node URL: {}", db_path, url); 58 | let agent = ureq::Agent::new_with_config( 59 | ureq::config::Config::builder() 60 | .max_response_header_size(usize::MAX) // Disabled as a workaround 61 | .build(), 62 | ); 63 | let client = client::Client::new(agent, url); 64 | let genesis_hash = client.get_blockhash_by_height(0)?; 65 | 66 | let store = db::Store::open(db_path)?; 67 | let chain = chain::Chain::new(store.headers()?); 68 | if let Some(indexed_genesis) = chain.genesis() { 69 | if indexed_genesis.hash() != genesis_hash { 70 | return Err(Error::ChainMismatch(indexed_genesis.hash(), genesis_hash)); 71 | } 72 | info!( 73 | "block={} height={} headers loaded", 74 | chain.tip_hash().unwrap(), 75 | chain.tip_height().unwrap() 76 | ); 77 | } 78 | Ok(Index { 79 | genesis_hash, 80 | chain, 81 | client, 82 | store, 83 | }) 84 | } 85 | 86 | pub fn open_default(network: cli::Network) -> Result { 87 | let bitcoin_network: bitcoin::Network = network.into(); 88 | let default_db_path = format!("db/{bitcoin_network}"); 89 | let default_rpc_port = match network { 90 | cli::Network::Bitcoin => 8332, 91 | cli::Network::Testnet => 18332, 92 | cli::Network::Testnet4 => 48332, 93 | cli::Network::Signet => 38332, 94 | cli::Network::Regtest => 18443, 95 | }; 96 | let default_rest_url = format!("http://localhost:{}", default_rpc_port); 97 | Self::open(default_db_path, default_rest_url) 98 | } 99 | 100 | fn drop_tip(&mut self) -> Result { 101 | let stale = self.chain.pop().expect("cannot drop tip of an empty chain"); 102 | let block_bytes = self.client.get_block_bytes(stale.hash())?; 103 | let spent_bytes = self.client.get_spent_bytes(stale.hash())?; 104 | let mut builder = index::Builder::new(&self.chain); 105 | builder.index(stale.hash(), &block_bytes, &spent_bytes)?; 106 | self.store.delete(&builder.into_batches())?; 107 | Ok(stale.hash()) 108 | } 109 | 110 | pub fn sync_chain(&mut self, limit: usize) -> Result { 111 | let mut stats = Stats::new( 112 | self.chain 113 | .tip_hash() 114 | .unwrap_or_else(bitcoin::BlockHash::all_zeros), 115 | ); 116 | let t = std::time::Instant::now(); 117 | 118 | let headers = loop { 119 | let blockhash = self.chain.tip_hash().unwrap_or(self.genesis_hash); 120 | let headers = self.client.get_headers(blockhash, limit)?; 121 | if let Some(first) = headers.first() { 122 | // skip first response header (when asking for non-genesis block) 123 | let skip_first = Some(first.block_hash()) == self.chain.tip_hash(); 124 | break headers.into_iter().skip(if skip_first { 1 } else { 0 }); 125 | } 126 | warn!( 127 | "block={} height={} was rolled back", 128 | blockhash, 129 | self.chain.tip_height().unwrap(), 130 | ); 131 | assert_eq!(blockhash, self.drop_tip()?); 132 | }; 133 | 134 | let mut builder = index::Builder::new(&self.chain); 135 | for header in headers { 136 | let blockhash = header.block_hash(); 137 | if self.chain.tip_hash() == Some(blockhash) { 138 | continue; // skip first header from response 139 | } 140 | // TODO: can be done concurrently 141 | let block_bytes = self.client.get_block_bytes(blockhash)?; 142 | let spent_bytes = self.client.get_spent_bytes(blockhash)?; 143 | builder.index(blockhash, &block_bytes, &spent_bytes)?; 144 | 145 | stats.tip = blockhash; 146 | stats.size_read += block_bytes.len(); 147 | stats.size_read += spent_bytes.len(); 148 | stats.indexed_blocks += 1; 149 | } 150 | let batches = builder.into_batches(); 151 | self.store.write(&batches)?; 152 | for batch in batches { 153 | self.chain.add(batch.header); 154 | } 155 | 156 | stats.elapsed = t.elapsed(); 157 | if stats.indexed_blocks > 0 { 158 | self.store.flush()?; 159 | info!( 160 | "block={} height={}: indexed {} blocks, {:.3}[MB], dt = {:.3}[s]: {:.3} [ms/block], {:.3} [MB/block], {:.3} [MB/s]", 161 | self.chain.tip_hash().unwrap(), 162 | self.chain.tip_height().unwrap(), 163 | stats.indexed_blocks, 164 | stats.size_read as f64 / (1e6), 165 | stats.elapsed.as_secs_f64(), 166 | stats.elapsed.as_secs_f64() * 1e3 / (stats.indexed_blocks as f64), 167 | stats.size_read as f64 / (1e6 * stats.indexed_blocks as f64), 168 | stats.size_read as f64 / (1e6 * stats.elapsed.as_secs_f64()), 169 | ); 170 | } else { 171 | self.store.start_compactions()?; 172 | } 173 | Ok(stats) 174 | } 175 | 176 | fn find_locations( 177 | &self, 178 | script_hash: &index::ScriptHash, 179 | from: index::TxNum, 180 | ) -> Result, Error> { 181 | Ok(self 182 | .store 183 | .scan(script_hash, from)? 184 | // chain and store must be in sync 185 | .map(|txnum| self.chain.find_by_txnum(&txnum).expect("invalid position"))) 186 | } 187 | 188 | fn get_tx_bytes(&self, location: &Location) -> Result, Error> { 189 | Ok(self 190 | .client 191 | .get_tx_bytes_from_block(location.indexed_header.hash(), location.offset)?) 192 | } 193 | 194 | fn chain(&self) -> &Chain { 195 | &self.chain 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /bindex-lib/src/chain.rs: -------------------------------------------------------------------------------- 1 | use crate::index; 2 | 3 | use bitcoin::{hashes::Hash, BlockHash}; 4 | 5 | #[derive(thiserror::Error, Debug)] 6 | pub enum Reorg { 7 | #[error("missing block={0} at height={1}")] 8 | Missing(bitcoin::BlockHash, usize), 9 | 10 | #[error("stale block={0} at height={1}")] 11 | Stale(bitcoin::BlockHash, usize), 12 | } 13 | 14 | pub struct Chain { 15 | rows: Vec, 16 | } 17 | 18 | impl Chain { 19 | pub fn new(rows: Vec) -> Self { 20 | let mut block_hash = bitcoin::BlockHash::all_zeros(); 21 | for row in &rows { 22 | assert_eq!(row.header().prev_blockhash, block_hash); 23 | block_hash = row.hash(); 24 | } 25 | Self { rows } 26 | } 27 | 28 | pub fn tip_hash(&self) -> Option { 29 | self.rows.last().map(index::Header::hash) 30 | } 31 | 32 | pub fn tip_height(&self) -> Option { 33 | self.rows.len().checked_sub(1) 34 | } 35 | 36 | pub fn next_txnum(&self) -> index::TxNum { 37 | self.rows 38 | .last() 39 | .map_or_else(index::TxNum::default, index::Header::next_txnum) 40 | } 41 | 42 | pub fn add(&mut self, row: index::Header) { 43 | assert_eq!( 44 | row.header().prev_blockhash, 45 | self.tip_hash().unwrap_or_else(BlockHash::all_zeros) 46 | ); 47 | self.rows.push(row) 48 | } 49 | 50 | pub fn pop(&mut self) -> Option { 51 | self.rows.pop() 52 | } 53 | 54 | pub fn genesis(&self) -> Option<&index::Header> { 55 | self.rows.first() 56 | } 57 | 58 | pub fn get_header(&self, hash: BlockHash, height: usize) -> Result<&index::Header, Reorg> { 59 | let header = self.rows.get(height).ok_or(Reorg::Missing(hash, height))?; 60 | if header.hash() == hash { 61 | Ok(header) 62 | } else { 63 | Err(Reorg::Stale(hash, height)) 64 | } 65 | } 66 | 67 | pub fn find_by_txnum(&self, txnum: &index::TxNum) -> Option { 68 | let height = match self 69 | .rows 70 | .binary_search_by_key(txnum, index::Header::next_txnum) 71 | { 72 | Ok(i) => i + 1, // hitting exactly a block boundary txnum -> next block 73 | Err(i) => i, 74 | }; 75 | 76 | let indexed_header = self.rows.get(height)?; 77 | let prev_pos = self 78 | .rows 79 | .get(height - 1) 80 | .map_or_else(index::TxNum::default, index::Header::next_txnum); 81 | 82 | assert!( 83 | txnum >= &prev_pos, 84 | "binary search failed to find the correct position" 85 | ); 86 | let offset = txnum.offset_from(prev_pos).unwrap(); 87 | Some(Location { 88 | height, 89 | offset, 90 | indexed_header, 91 | }) 92 | } 93 | } 94 | 95 | #[derive(PartialEq, Eq, PartialOrd, Clone, Copy)] 96 | pub struct Location<'a> { 97 | pub height: usize, // block height 98 | pub offset: u64, // tx position within its block 99 | pub indexed_header: &'a index::Header, 100 | } 101 | 102 | impl Ord for Location<'_> { 103 | fn cmp(&self, other: &Self) -> std::cmp::Ordering { 104 | (self.height, self.offset).cmp(&(other.height, other.offset)) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /bindex-lib/src/cli.rs: -------------------------------------------------------------------------------- 1 | use clap::ValueEnum; 2 | 3 | #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Debug)] 4 | pub enum Network { 5 | Bitcoin, 6 | Testnet, 7 | Testnet4, 8 | Signet, 9 | Regtest, 10 | } 11 | 12 | impl From for bitcoin::Network { 13 | fn from(value: Network) -> Self { 14 | match value { 15 | Network::Bitcoin => bitcoin::Network::Bitcoin, 16 | Network::Testnet => bitcoin::Network::Testnet, 17 | Network::Testnet4 => bitcoin::Network::Testnet4, 18 | Network::Signet => bitcoin::Network::Signet, 19 | Network::Regtest => bitcoin::Network::Regtest, 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /bindex-lib/src/client.rs: -------------------------------------------------------------------------------- 1 | use std::{io::ErrorKind, time::Duration}; 2 | 3 | use bitcoin::{ 4 | block::Header, 5 | consensus::{deserialize, Decodable}, 6 | BlockHash, 7 | }; 8 | use log::*; 9 | 10 | use crate::index; 11 | 12 | #[derive(thiserror::Error, Debug)] 13 | pub enum Error { 14 | #[error("request failed: {0}")] 15 | Http(#[from] ureq::Error), 16 | 17 | #[error("reading response failed: {0}")] 18 | Io(#[from] std::io::Error), 19 | 20 | #[error("decoding failed: {0}")] 21 | Decoding(#[from] bitcoin::consensus::encode::Error), 22 | } 23 | 24 | pub struct Client { 25 | agent: ureq::Agent, 26 | url: String, 27 | } 28 | 29 | impl Client { 30 | pub fn new>(agent: ureq::Agent, url: T) -> Self { 31 | Self { 32 | agent, 33 | url: url.into(), 34 | } 35 | } 36 | 37 | fn get_bytes(&self, url: &str) -> Result, Error> { 38 | let mut iter = 0; 39 | let err = loop { 40 | iter += 1; 41 | let req = self.agent.get(url); 42 | debug!("=> {:?}", req); 43 | let res = req.call(); 44 | debug!("<= {:?}", res); 45 | let err = match res { 46 | Ok(resp) => return Ok(resp.into_body().read_to_vec()?), 47 | Err(err) => err, 48 | }; 49 | if iter > 100 { 50 | break err; 51 | } 52 | match &err { 53 | ureq::Error::StatusCode(503) => (), 54 | ureq::Error::Io(e) if e.kind() == ErrorKind::ConnectionRefused => (), 55 | _ => break err, // non-retriable error 56 | } 57 | warn!("unavailable {}: {:?}", url, err); 58 | std::thread::sleep(Duration::from_secs(1)); 59 | }; 60 | error!("GET {} failed: {:?}", url, err); 61 | Err(Error::Http(err)) 62 | } 63 | 64 | pub fn get_blockhash_by_height(&self, height: usize) -> Result { 65 | let url = format!("{}/rest/blockhashbyheight/{}.bin", self.url, height); 66 | let data = self.get_bytes(&url)?; 67 | Ok(deserialize(&data)?) 68 | } 69 | 70 | pub fn get_headers(&self, hash: BlockHash, limit: usize) -> Result, Error> { 71 | let url = format!("{}/rest/headers/{}/{}.bin", self.url, limit + 1, hash); 72 | let data = self.get_bytes(&url)?; 73 | assert_eq!(data.len() % Header::SIZE, 0); 74 | let count = data.len() / Header::SIZE; 75 | 76 | // the first header should correspond to `hash` 77 | let mut headers = Vec::with_capacity(count); 78 | let mut r = bitcoin::io::Cursor::new(data); 79 | for _ in 0..count { 80 | let header = Header::consensus_decode_from_finite_reader(&mut r)?; 81 | headers.push(header); 82 | } 83 | Ok(headers) 84 | } 85 | 86 | pub fn get_block_bytes(&self, hash: BlockHash) -> Result { 87 | let url = format!("{}/rest/block/{}.bin", self.url, hash); 88 | let data = self.get_bytes(&url)?; 89 | Ok(index::BlockBytes::new(data)) 90 | } 91 | 92 | pub fn get_spent_bytes(&self, hash: BlockHash) -> Result { 93 | let url = format!("{}/rest/spenttxouts/{}.bin", self.url, hash); 94 | let data = self.get_bytes(&url)?; 95 | Ok(index::SpentBytes::new(data)) 96 | } 97 | 98 | pub fn get_tx_bytes_from_block(&self, hash: BlockHash, offset: u64) -> Result, Error> { 99 | let url = format!("{}/rest/txfromblock/{}-{}.bin", self.url, hash, offset); 100 | self.get_bytes(&url) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /bindex-lib/src/db.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use crate::index; 4 | 5 | use log::*; 6 | 7 | pub struct Store { 8 | db: rocksdb::DB, 9 | compacting: bool, 10 | } 11 | 12 | fn default_opts() -> rocksdb::Options { 13 | let mut opts = rocksdb::Options::default(); 14 | opts.create_if_missing(true); 15 | opts.create_missing_column_families(true); 16 | opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); 17 | opts.set_compression_type(rocksdb::DBCompressionType::Zstd); 18 | opts.set_max_open_files(256); 19 | opts.set_keep_log_file_num(10); 20 | opts.set_disable_auto_compactions(true); 21 | 22 | let parallelism = std::thread::available_parallelism() 23 | .ok() 24 | .and_then(|v| u16::try_from(v.get()).ok()) 25 | .unwrap_or(2) 26 | .clamp(1, 8); 27 | opts.increase_parallelism(parallelism.into()); 28 | opts.set_max_subcompactions(parallelism.into()); 29 | opts 30 | } 31 | 32 | const HEADERS_CF: &str = "headers"; 33 | const SCRIPT_HASH_CF: &str = "script_hash"; 34 | 35 | const COLUMN_FAMILIES: &[&str] = &[HEADERS_CF, SCRIPT_HASH_CF]; 36 | 37 | fn cf_descriptors( 38 | opts: &rocksdb::Options, 39 | ) -> impl IntoIterator + '_ { 40 | COLUMN_FAMILIES 41 | .iter() 42 | .map(|&name| rocksdb::ColumnFamilyDescriptor::new(name, opts.clone())) 43 | } 44 | 45 | impl Store { 46 | pub fn open(path: impl AsRef) -> Result { 47 | let opts = default_opts(); 48 | let db = rocksdb::DB::open_cf_descriptors(&opts, path, cf_descriptors(&opts))?; 49 | 50 | let store = Self { 51 | db, 52 | compacting: false, 53 | }; 54 | for &cf_name in COLUMN_FAMILIES { 55 | let cf = store.cf(cf_name); 56 | let metadata = store.db.get_column_family_metadata_cf(cf); 57 | info!( 58 | "CF {}: {} files, {:.6} MBs", 59 | cf_name, 60 | metadata.file_count, 61 | metadata.size as f64 / 1e6 62 | ); 63 | } 64 | Ok(store) 65 | } 66 | 67 | fn cf(&self, name: &str) -> &rocksdb::ColumnFamily { 68 | self.db 69 | .cf_handle(name) 70 | .unwrap_or_else(|| panic!("missing CF: {}", name)) 71 | } 72 | 73 | pub fn write(&self, batches: &[index::Batch]) -> Result<(), rocksdb::Error> { 74 | if batches.is_empty() { 75 | return Ok(()); 76 | } 77 | let mut write_batch = rocksdb::WriteBatch::default(); 78 | let cf = self.cf(SCRIPT_HASH_CF); 79 | let mut script_hash_rows = vec![]; 80 | for batch in batches { 81 | script_hash_rows.extend( 82 | batch 83 | .script_hash_rows 84 | .iter() 85 | .map(index::ScriptHashPrefixRow::key), 86 | ); 87 | } 88 | script_hash_rows.sort_unstable(); 89 | for row in script_hash_rows { 90 | write_batch.put_cf(cf, row, b""); 91 | } 92 | 93 | let cf = self.cf(HEADERS_CF); 94 | for batch in batches { 95 | let (key, value) = batch.header.serialize(); 96 | write_batch.put_cf(cf, key, value); 97 | } 98 | 99 | let mut opts = rocksdb::WriteOptions::default(); 100 | opts.disable_wal(false); 101 | self.db.write_opt(write_batch, &opts)?; 102 | Ok(()) 103 | } 104 | 105 | pub fn delete(&self, batches: &[index::Batch]) -> Result<(), rocksdb::Error> { 106 | let mut write_batch = rocksdb::WriteBatch::default(); 107 | let cf = self.cf(SCRIPT_HASH_CF); 108 | let mut script_hash_rows = vec![]; 109 | for batch in batches { 110 | script_hash_rows.extend( 111 | batch 112 | .script_hash_rows 113 | .iter() 114 | .map(index::ScriptHashPrefixRow::key), 115 | ); 116 | } 117 | // ScriptHashPrefixRow::key contains txnum, so it is safe to delete 118 | script_hash_rows.sort_unstable(); 119 | for row in script_hash_rows { 120 | write_batch.delete_cf(cf, row); 121 | } 122 | 123 | let cf = self.cf(HEADERS_CF); 124 | // index::Header key is next_txnum, so it is safe to delete 125 | for batch in batches { 126 | let (key, _value) = batch.header.serialize(); 127 | write_batch.delete_cf(cf, key); 128 | } 129 | 130 | let mut opts = rocksdb::WriteOptions::default(); 131 | opts.disable_wal(true); 132 | self.db.write_opt(write_batch, &opts)?; 133 | Ok(()) 134 | } 135 | 136 | pub fn flush(&self) -> Result<(), rocksdb::Error> { 137 | let opts = rocksdb::FlushOptions::new(); 138 | for cf in COLUMN_FAMILIES { 139 | self.db.flush_cf_opt(self.cf(cf), &opts)?; 140 | } 141 | Ok(()) 142 | } 143 | 144 | pub fn start_compactions(&mut self) -> Result<(), rocksdb::Error> { 145 | if !self.compacting { 146 | const OPTION: (&str, &str) = ("disable_auto_compactions", "false"); 147 | for &cf_name in COLUMN_FAMILIES { 148 | let cf = self.cf(cf_name); 149 | self.db.set_options_cf(cf, &[OPTION])?; 150 | } 151 | info!("started auto compactions"); 152 | self.compacting = true; 153 | } 154 | Ok(()) 155 | } 156 | 157 | pub fn scan( 158 | &self, 159 | script_hash: &index::ScriptHash, 160 | from: index::TxNum, 161 | ) -> Result, rocksdb::Error> { 162 | let cf = self.cf(SCRIPT_HASH_CF); 163 | let mut positions = Vec::new(); 164 | 165 | let prefix = index::ScriptHashPrefix::new(script_hash); 166 | let start = index::ScriptHashPrefixRow::new(prefix, from); 167 | let mode = rocksdb::IteratorMode::From(start.key(), rocksdb::Direction::Forward); 168 | for kv in self.db.iterator_cf(cf, mode) { 169 | let (key, _) = kv?; 170 | if !key.starts_with(prefix.as_bytes()) { 171 | break; 172 | } 173 | let row = index::ScriptHashPrefixRow::from_bytes(key[..].try_into().unwrap()); 174 | assert!(row.txnum() >= from); 175 | positions.push(row.txnum()); 176 | } 177 | Ok(positions.into_iter()) 178 | } 179 | 180 | pub fn headers(&self) -> Result, rocksdb::Error> { 181 | let cf = self.cf(HEADERS_CF); 182 | let mut result = vec![]; 183 | for kv in self.db.iterator_cf(cf, rocksdb::IteratorMode::Start) { 184 | let (key, value) = kv?; 185 | let row = index::Header::deserialize(( 186 | key[..].try_into().unwrap(), 187 | value[..].try_into().unwrap(), 188 | )); 189 | result.push(row) 190 | } 191 | Ok(result) 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /bindex-lib/src/index.rs: -------------------------------------------------------------------------------- 1 | use std::ops::ControlFlow; 2 | 3 | use bitcoin::{consensus::Encodable, hashes::Hash, BlockHash}; 4 | use bitcoin_slices::{bsl, Parse, Visit}; 5 | 6 | use crate::chain::Chain; 7 | 8 | #[derive(thiserror::Error, Debug)] 9 | pub enum Error { 10 | #[error("decoding failed: {0}")] 11 | Decode(#[from] bitcoin::consensus::encode::Error), 12 | 13 | #[error("parsing failed: {0:?}")] 14 | Parse(bitcoin_slices::Error), 15 | 16 | #[error("{0} bytes were not parsed")] 17 | Leftover(usize), 18 | } 19 | 20 | bitcoin::hashes::hash_newtype! { 21 | /// https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#script-hashes 22 | #[hash_newtype(backward)] 23 | pub struct ScriptHash(bitcoin::hashes::sha256::Hash); 24 | } 25 | 26 | impl ScriptHash { 27 | pub fn new(script: &bitcoin::Script) -> Self { 28 | Self::hash(script.as_bytes()) 29 | } 30 | } 31 | 32 | #[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)] 33 | pub struct ScriptHashPrefix([u8; ScriptHashPrefix::LEN]); 34 | 35 | impl ScriptHashPrefix { 36 | const LEN: usize = 8; 37 | 38 | pub fn new(script_hash: &ScriptHash) -> Self { 39 | Self(script_hash[..ScriptHashPrefix::LEN].try_into().unwrap()) 40 | } 41 | 42 | pub fn as_bytes(&self) -> &[u8] { 43 | &self.0 44 | } 45 | } 46 | 47 | #[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord, Default)] 48 | pub struct TxNum(u64); 49 | 50 | impl TxNum { 51 | const LEN: usize = std::mem::size_of::(); 52 | 53 | pub fn offset_from(&self, base: TxNum) -> Option { 54 | self.0.checked_sub(base.0) 55 | } 56 | } 57 | 58 | #[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)] 59 | pub struct ScriptHashPrefixRow { 60 | key: [u8; ScriptHashPrefixRow::LEN], 61 | } 62 | 63 | impl ScriptHashPrefixRow { 64 | const LEN: usize = ScriptHashPrefix::LEN + TxNum::LEN; 65 | 66 | pub fn new(prefix: ScriptHashPrefix, txnum: TxNum) -> Self { 67 | let mut result = [0u8; ScriptHashPrefix::LEN + TxNum::LEN]; 68 | result[..ScriptHashPrefix::LEN].copy_from_slice(&prefix.0); 69 | result[ScriptHashPrefix::LEN..].copy_from_slice(&txnum.0.to_be_bytes()); 70 | Self { key: result } 71 | } 72 | 73 | pub fn key(&self) -> &[u8] { 74 | &self.key 75 | } 76 | 77 | pub fn from_bytes(key: [u8; Self::LEN]) -> Self { 78 | Self { key } 79 | } 80 | 81 | pub fn txnum(&self) -> TxNum { 82 | TxNum(u64::from_be_bytes( 83 | self.key[ScriptHashPrefix::LEN..].try_into().unwrap(), 84 | )) 85 | } 86 | } 87 | 88 | #[derive(Debug, PartialEq, Eq)] 89 | struct IndexVisitor<'a> { 90 | rows: &'a mut Vec, 91 | txnum: TxNum, 92 | } 93 | 94 | impl<'a> IndexVisitor<'a> { 95 | fn new(txnum: TxNum, rows: &'a mut Vec) -> Self { 96 | Self { txnum, rows } 97 | } 98 | 99 | fn add(&mut self, script: &bitcoin::Script) { 100 | if script.is_op_return() { 101 | // skip indexing unspendable outputs 102 | return; 103 | } 104 | let script_hash = ScriptHash::new(script); 105 | self.rows.push(ScriptHashPrefixRow::new( 106 | ScriptHashPrefix::new(&script_hash), 107 | self.txnum, 108 | )); 109 | } 110 | 111 | fn finish_tx(&mut self) { 112 | self.txnum.0 += 1; 113 | } 114 | } 115 | 116 | impl bitcoin_slices::Visitor for IndexVisitor<'_> { 117 | fn visit_tx_out(&mut self, _vout: usize, tx_out: &bsl::TxOut) -> ControlFlow<()> { 118 | self.add(bitcoin::Script::from_bytes(tx_out.script_pubkey())); 119 | ControlFlow::Continue(()) 120 | } 121 | 122 | fn visit_transaction(&mut self, _tx: &bsl::Transaction) -> ControlFlow<()> { 123 | // Updated after all txouts are scanned 124 | self.finish_tx(); 125 | ControlFlow::Continue(()) 126 | } 127 | } 128 | 129 | struct Spent; 130 | 131 | impl AsRef<[u8]> for Spent { 132 | fn as_ref(&self) -> &[u8] { 133 | &[] 134 | } 135 | } 136 | 137 | fn visit_spent<'a>( 138 | slice: &'a [u8], 139 | visit: &mut IndexVisitor, 140 | ) -> bitcoin_slices::SResult<'a, Spent> { 141 | let mut consumed = 0; 142 | let txs_count = bsl::scan_len(slice, &mut consumed)?; 143 | 144 | for _ in 0..txs_count { 145 | let outputs_count = bsl::scan_len(&slice[consumed..], &mut consumed)?; 146 | 147 | for _ in 0..outputs_count { 148 | let tx_out = bsl::TxOut::parse(&slice[consumed..])?; 149 | consumed += tx_out.consumed(); 150 | let script_pubkey = tx_out.parsed().script_pubkey(); 151 | visit.add(bitcoin::Script::from_bytes(script_pubkey)); 152 | } 153 | visit.finish_tx(); 154 | } 155 | 156 | Ok(bitcoin_slices::ParseResult::new(&slice[consumed..], Spent)) 157 | } 158 | 159 | fn add_block_rows( 160 | block: &BlockBytes, 161 | txnum: TxNum, 162 | rows: &mut Vec, 163 | ) -> Result { 164 | let mut visitor = IndexVisitor::new(txnum, rows); 165 | let res = bsl::Block::visit(&block.0, &mut visitor).map_err(Error::Parse)?; 166 | if !res.remaining().is_empty() { 167 | return Err(Error::Leftover(res.remaining().len())); 168 | } 169 | Ok(visitor.txnum) 170 | } 171 | 172 | fn add_spent_rows( 173 | spent: &SpentBytes, 174 | txnum: TxNum, 175 | rows: &mut Vec, 176 | ) -> Result { 177 | let mut visitor = IndexVisitor::new(txnum, rows); 178 | let res = visit_spent(&spent.0, &mut visitor).map_err(Error::Parse)?; 179 | if !res.remaining().is_empty() { 180 | return Err(Error::Leftover(res.remaining().len())); 181 | } 182 | Ok(visitor.txnum) 183 | } 184 | 185 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] 186 | pub struct Header { 187 | next_txnum: TxNum, 188 | hash: bitcoin::BlockHash, 189 | header: bitcoin::block::Header, 190 | } 191 | 192 | const BLOCK_HASH_LEN: usize = bitcoin::BlockHash::LEN; 193 | const BLOCK_HEADER_LEN: usize = bitcoin::block::Header::SIZE; 194 | 195 | type SerializedHeaderRow = ([u8; TxNum::LEN], [u8; BLOCK_HASH_LEN + BLOCK_HEADER_LEN]); 196 | 197 | impl Header { 198 | fn new(next_txnum: TxNum, hash: bitcoin::BlockHash, header: bitcoin::block::Header) -> Self { 199 | Self { 200 | next_txnum, 201 | hash, 202 | header, 203 | } 204 | } 205 | 206 | pub fn serialize(&self) -> SerializedHeaderRow { 207 | let key = self.next_txnum.0.to_be_bytes(); 208 | let mut value = [0u8; BLOCK_HASH_LEN + BLOCK_HEADER_LEN]; 209 | value[..BLOCK_HASH_LEN].copy_from_slice(self.hash.as_byte_array()); 210 | self.header 211 | .consensus_encode(&mut &mut value[BLOCK_HASH_LEN..]) 212 | .unwrap(); 213 | (key, value) 214 | } 215 | 216 | pub fn deserialize((key, value): SerializedHeaderRow) -> Self { 217 | Self { 218 | next_txnum: TxNum(u64::from_be_bytes(key)), 219 | hash: BlockHash::from_byte_array(value[..BLOCK_HASH_LEN].try_into().unwrap()), 220 | header: bitcoin::consensus::encode::deserialize(&value[BLOCK_HASH_LEN..]).unwrap(), 221 | } 222 | } 223 | 224 | pub fn next_txnum(&self) -> TxNum { 225 | self.next_txnum 226 | } 227 | 228 | pub fn hash(&self) -> BlockHash { 229 | self.hash 230 | } 231 | 232 | pub fn header(&self) -> &bitcoin::block::Header { 233 | &self.header 234 | } 235 | } 236 | 237 | pub struct BlockBytes(Vec); 238 | 239 | impl BlockBytes { 240 | pub fn new(data: Vec) -> Self { 241 | BlockBytes(data) 242 | } 243 | 244 | fn header(&self) -> &[u8] { 245 | &self.0[..BLOCK_HEADER_LEN] 246 | } 247 | 248 | pub fn len(&self) -> usize { 249 | self.0.len() 250 | } 251 | } 252 | 253 | pub struct SpentBytes(Vec); 254 | 255 | impl SpentBytes { 256 | pub fn new(data: Vec) -> Self { 257 | SpentBytes(data) 258 | } 259 | 260 | pub fn len(&self) -> usize { 261 | self.0.len() 262 | } 263 | } 264 | 265 | pub struct Batch { 266 | pub script_hash_rows: Vec, 267 | pub header: Header, 268 | } 269 | 270 | impl Batch { 271 | fn build( 272 | hash: BlockHash, 273 | txnum: TxNum, 274 | block: &BlockBytes, 275 | spent: &SpentBytes, 276 | ) -> Result { 277 | let mut script_hash_rows = vec![]; 278 | let txnum = { 279 | let num1 = add_block_rows(block, txnum, &mut script_hash_rows)?; 280 | let num2 = add_spent_rows(spent, txnum, &mut script_hash_rows)?; 281 | assert_eq!(num1, num2); // both must have the same number of transactions 282 | num1 283 | }; 284 | let header = Header::new( 285 | txnum, 286 | hash, 287 | bitcoin::consensus::encode::deserialize(block.header())?, 288 | ); 289 | Ok(Batch { 290 | script_hash_rows, 291 | header, 292 | }) 293 | } 294 | } 295 | 296 | pub struct Builder { 297 | batches: Vec, 298 | next_txnum: TxNum, 299 | tip: bitcoin::BlockHash, 300 | } 301 | 302 | impl Builder { 303 | pub fn new(chain: &Chain) -> Self { 304 | Self { 305 | next_txnum: chain.next_txnum(), 306 | batches: vec![], 307 | tip: chain 308 | .tip_hash() 309 | .unwrap_or_else(bitcoin::BlockHash::all_zeros), 310 | } 311 | } 312 | 313 | pub fn index( 314 | &mut self, 315 | hash: bitcoin::BlockHash, 316 | block_bytes: &BlockBytes, 317 | spent_bytes: &SpentBytes, 318 | ) -> Result<(), Error> { 319 | let batch = Batch::build(hash, self.next_txnum, block_bytes, spent_bytes)?; 320 | assert_eq!(batch.header.header().prev_blockhash, self.tip); 321 | self.next_txnum = batch.header.next_txnum(); 322 | self.tip = batch.header.hash; 323 | self.batches.push(batch); 324 | Ok(()) 325 | } 326 | 327 | pub fn into_batches(self) -> Vec { 328 | self.batches 329 | } 330 | } 331 | 332 | #[cfg(test)] 333 | mod tests { 334 | use bitcoin::consensus::{deserialize, encode::Decodable}; 335 | use hex_lit::hex; 336 | 337 | use super::*; 338 | 339 | // Block 100000 340 | const BLOCK_HEX: &str = "0100000050120119172a610421a6c3011dd330d9df07b63616c2cc1f1cd00200000000006657a9252aacd5c0b2940996ecff952228c3067cc38d4885efb5a4ac4247e9f337221b4d4c86041b0f2b57100401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff08044c86041b020602ffffffff0100f2052a010000004341041b0e8c2567c12536aa13357b79a073dc4444acb83c4ec7a0e2f99dd7457516c5817242da796924ca4e99947d087fedf9ce467cb9f7c6287078f801df276fdf84ac000000000100000001032e38e9c0a84c6046d687d10556dcacc41d275ec55fc00779ac88fdf357a187000000008c493046022100c352d3dd993a981beba4a63ad15c209275ca9470abfcd57da93b58e4eb5dce82022100840792bc1f456062819f15d33ee7055cf7b5ee1af1ebcc6028d9cdb1c3af7748014104f46db5e9d61a9dc27b8d64ad23e7383a4e6ca164593c2527c038c0857eb67ee8e825dca65046b82c9331586c82e0fd1f633f25f87c161bc6f8a630121df2b3d3ffffffff0200e32321000000001976a914c398efa9c392ba6013c5e04ee729755ef7f58b3288ac000fe208010000001976a914948c765a6914d43f2a7ac177da2c2f6b52de3d7c88ac000000000100000001c33ebff2a709f13d9f9a7569ab16a32786af7d7e2de09265e41c61d078294ecf010000008a4730440220032d30df5ee6f57fa46cddb5eb8d0d9fe8de6b342d27942ae90a3231e0ba333e02203deee8060fdc70230a7f5b4ad7d7bc3e628cbe219a886b84269eaeb81e26b4fe014104ae31c31bf91278d99b8377a35bbce5b27d9fff15456839e919453fc7b3f721f0ba403ff96c9deeb680e5fd341c0fc3a7b90da4631ee39560639db462e9cb850fffffffff0240420f00000000001976a914b0dcbf97eabf4404e31d952477ce822dadbe7e1088acc060d211000000001976a9146b1281eec25ab4e1e0793ff4e08ab1abb3409cd988ac0000000001000000010b6072b386d4a773235237f64c1126ac3b240c84b917a3909ba1c43ded5f51f4000000008c493046022100bb1ad26df930a51cce110cf44f7a48c3c561fd977500b1ae5d6b6fd13d0b3f4a022100c5b42951acedff14abba2736fd574bdb465f3e6f8da12e2c5303954aca7f78f3014104a7135bfe824c97ecc01ec7d7e336185c81e2aa2c41ab175407c09484ce9694b44953fcb751206564a9c24dd094d42fdbfdd5aad3e063ce6af4cfaaea4ea14fbbffffffff0140420f00000000001976a91439aa3d569e06a1d7926dc4be1193c99bf2eb9ee088ac00000000"; 341 | const SPENT_HEX: &str = "04000100f2052a010000001976a91471d7dd96d9edda09180fe9d57a477b5acc9cad1188ac0100a3e111000000001976a91435fbee6a3bf8d99f17724ec54787567393a8a6b188ac0140420f00000000001976a914c4eb47ecfdcf609a1848ee79acc2fa49d3caad7088ac"; 342 | 343 | #[test] 344 | fn test_index_block() -> Result<(), Error> { 345 | let block_bytes = BlockBytes(hex!(BLOCK_HEX).to_vec()); 346 | let spent_bytes = SpentBytes(hex!(SPENT_HEX).to_vec()); 347 | let txnum = TxNum(10); 348 | 349 | let mut block_rows = vec![]; 350 | assert_eq!( 351 | add_block_rows(&block_bytes, txnum, &mut block_rows)?, 352 | TxNum(14) 353 | ); 354 | 355 | assert_eq!( 356 | block_rows, 357 | vec![ 358 | ScriptHashPrefixRow::new(ScriptHashPrefix(hex!("e2151d493a1f9999")), TxNum(10)), 359 | ScriptHashPrefixRow::new(ScriptHashPrefix(hex!("050b00fb9d5f7a63")), TxNum(11)), 360 | ScriptHashPrefixRow::new(ScriptHashPrefix(hex!("b5a1091a739a6aba")), TxNum(11)), 361 | ScriptHashPrefixRow::new(ScriptHashPrefix(hex!("03b0bfb44fd9d852")), TxNum(12)), 362 | ScriptHashPrefixRow::new(ScriptHashPrefix(hex!("0faa9934b57389f2")), TxNum(12)), 363 | ScriptHashPrefixRow::new(ScriptHashPrefix(hex!("4a569bc2092bcaf9")), TxNum(13)) 364 | ] 365 | ); 366 | 367 | let mut spent_rows = vec![]; 368 | assert_eq!( 369 | add_spent_rows(&spent_bytes, txnum, &mut spent_rows)?, 370 | TxNum(14) 371 | ); 372 | 373 | assert_eq!( 374 | spent_rows, 375 | vec![ 376 | ScriptHashPrefixRow::new(ScriptHashPrefix(hex!("4d5bea28470692cd")), TxNum(11)), 377 | ScriptHashPrefixRow::new(ScriptHashPrefix(hex!("e9b09b065b5f43c2")), TxNum(12)), 378 | ScriptHashPrefixRow::new(ScriptHashPrefix(hex!("2e7cdb30882b427d")), TxNum(13)), 379 | ] 380 | ); 381 | 382 | // Verify spent outputs indexing 383 | let mut test_spent_rows = vec![]; 384 | assert_eq!( 385 | decode_spent(&spent_bytes.0, txnum, &mut test_spent_rows)?, 386 | TxNum(14) 387 | ); 388 | assert_eq!(test_spent_rows, spent_rows); 389 | 390 | // Verify public interface 391 | let block: bitcoin::Block = deserialize(&block_bytes.0).unwrap(); 392 | let batch = Batch::build(block.block_hash(), txnum, &block_bytes, &spent_bytes)?; 393 | 394 | assert_eq!(batch.header.next_txnum(), TxNum(14)); 395 | assert_eq!(batch.header.hash(), block.block_hash()); 396 | assert_eq!(batch.script_hash_rows, [block_rows, spent_rows].concat()); 397 | 398 | Ok(()) 399 | } 400 | 401 | fn decode_spent( 402 | buf: &[u8], 403 | txnum: TxNum, 404 | rows: &mut Vec, 405 | ) -> Result { 406 | let mut visitor = IndexVisitor::new(txnum, rows); 407 | let mut r = bitcoin::io::Cursor::new(buf); 408 | let txs_count = bitcoin::VarInt::consensus_decode_from_finite_reader(&mut r)?.0; 409 | for _ in 0..txs_count { 410 | let outputs_count = bitcoin::VarInt::consensus_decode_from_finite_reader(&mut r)?.0; 411 | for _ in 0..outputs_count { 412 | let output = bitcoin::TxOut::consensus_decode_from_finite_reader(&mut r)?; 413 | visitor.add(&output.script_pubkey); 414 | } 415 | visitor.finish_tx(); 416 | } 417 | let pos: usize = r.position().try_into().unwrap(); 418 | if pos == buf.len() { 419 | Ok(visitor.txnum) 420 | } else { 421 | Err(Error::Leftover(buf.len() - pos)) 422 | } 423 | } 424 | 425 | #[test] 426 | fn test_serde_row() { 427 | let txnum = TxNum(0x123456789ABCDEF0); 428 | let row = ScriptHashPrefixRow::new(ScriptHashPrefix([1, 2, 3, 4, 5, 6, 7, 8]), txnum); 429 | assert_eq!(row.txnum(), txnum); 430 | let data = row.key; 431 | assert_eq!(data, hex!("0102030405060708123456789abcdef0")); 432 | assert_eq!(ScriptHashPrefixRow::from_bytes(data), row); 433 | } 434 | 435 | #[test] 436 | fn test_map_txnum() { 437 | let txnum = [10, 20, 30, 40]; 438 | 439 | assert_eq!(txnum.binary_search(&0), Err(0)); 440 | assert_eq!(txnum.binary_search(&1), Err(0)); 441 | assert_eq!(txnum.binary_search(&9), Err(0)); 442 | assert_eq!(txnum.binary_search(&10), Ok(0)); 443 | assert_eq!(txnum.binary_search(&11), Err(1)); 444 | assert_eq!(txnum.binary_search(&19), Err(1)); 445 | assert_eq!(txnum.binary_search(&20), Ok(1)); 446 | assert_eq!(txnum.binary_search(&21), Err(2)); 447 | assert_eq!(txnum.binary_search(&29), Err(2)); 448 | assert_eq!(txnum.binary_search(&30), Ok(2)); 449 | assert_eq!(txnum.binary_search(&31), Err(3)); 450 | assert_eq!(txnum.binary_search(&39), Err(3)); 451 | assert_eq!(txnum.binary_search(&40), Ok(3)); 452 | assert_eq!(txnum.binary_search(&41), Err(4)); 453 | } 454 | } 455 | -------------------------------------------------------------------------------- /bindex-lib/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod address; 2 | pub use bitcoin; 3 | mod chain; 4 | pub mod cli; 5 | mod client; 6 | mod db; 7 | mod index; 8 | -------------------------------------------------------------------------------- /electrum/LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020, The Electrum developers 2 | Copyright (c) 2016-2020, Neil Booth 3 | 4 | All rights reserved. 5 | 6 | The MIT License (MIT) 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining 9 | a copy of this software and associated documentation files (the 10 | "Software"), to deal in the Software without restriction, including 11 | without limitation the rights to use, copy, modify, merge, publish, 12 | distribute, sublicense, and/or sell copies of the Software, and to 13 | permit persons to whom the Software is furnished to do so, subject to 14 | the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be 17 | included in all copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 22 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 23 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 24 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /electrum/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/romanz/bindex-rs/a587c24ef08eeacef99ab444e42fcb4e57d449a2/electrum/__init__.py -------------------------------------------------------------------------------- /electrum/merkle.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, Neil Booth 2 | # 3 | # All rights reserved. 4 | # 5 | # The MIT License (MIT) 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining 8 | # a copy of this software and associated documentation files (the 9 | # "Software"), to deal in the Software without restriction, including 10 | # without limitation the rights to use, copy, modify, merge, publish, 11 | # distribute, sublicense, and/or sell copies of the Software, and to 12 | # permit persons to whom the Software is furnished to do so, subject to 13 | # the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be 16 | # included in all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 21 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 22 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 24 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 | # and warranty status of this software. 26 | 27 | """Merkle trees, branches, proofs and roots.""" 28 | 29 | from math import ceil, log 30 | from typing import Optional, Callable, Tuple, Iterable, List 31 | 32 | import hashlib 33 | 34 | _sha256 = hashlib.sha256 35 | 36 | 37 | def sha256(x): 38 | """Simple wrapper of hashlib sha256.""" 39 | return _sha256(x).digest() 40 | 41 | 42 | def double_sha256(x): 43 | """SHA-256 of SHA-256, as used extensively in bitcoin.""" 44 | return sha256(sha256(x)) 45 | 46 | 47 | class Merkle: 48 | """Perform merkle tree calculations on binary hashes using a given hash 49 | function. 50 | 51 | If the hash count is not even, the final hash is repeated when 52 | calculating the next merkle layer up the tree. 53 | """ 54 | 55 | def __init__(self, hash_func: Callable[[bytes], bytes] = double_sha256): 56 | self.hash_func = hash_func 57 | 58 | def branch_length(self, hash_count: int) -> int: 59 | """Return the length of a merkle branch given the number of hashes.""" 60 | if not isinstance(hash_count, int): 61 | raise TypeError("hash_count must be an integer") 62 | if hash_count < 1: 63 | raise ValueError("hash_count must be at least 1") 64 | return ceil(log(hash_count, 2)) 65 | 66 | def branch_and_root( 67 | self, 68 | hashes: Iterable[bytes], 69 | index: int, 70 | length: Optional[int] = None, 71 | ) -> Tuple[List[bytes], bytes]: 72 | """Return a (merkle branch, merkle_root) pair given hashes, and the 73 | index of one of those hashes. 74 | """ 75 | hashes = list(hashes) 76 | if not isinstance(index, int): 77 | raise TypeError("index must be an integer") 78 | # This also asserts hashes is not empty 79 | if not 0 <= index < len(hashes): 80 | raise ValueError("index out of range") 81 | natural_length = self.branch_length(len(hashes)) 82 | if length is None: 83 | length = natural_length 84 | else: 85 | if not isinstance(length, int): 86 | raise TypeError("length must be an integer") 87 | if length < natural_length: 88 | raise ValueError("length out of range") 89 | 90 | hash_func = self.hash_func 91 | branch = [] 92 | for _ in range(length): 93 | if len(hashes) & 1: 94 | hashes.append(hashes[-1]) 95 | branch.append(hashes[index ^ 1]) 96 | index >>= 1 97 | hashes = [ 98 | hash_func(hashes[n] + hashes[n + 1]) for n in range(0, len(hashes), 2) 99 | ] 100 | 101 | return branch, hashes[0] 102 | -------------------------------------------------------------------------------- /electrum/server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016-2018, Neil Booth 2 | # 3 | # All rights reserved. 4 | # 5 | # See the file "LICENCE" for information about the copyright 6 | # and warranty status of this software. 7 | 8 | """Classes for local RPC server and remote client TCP/SSL servers.""" 9 | 10 | import aiohttp 11 | import asyncio 12 | import functools 13 | import itertools 14 | import logging 15 | import sqlite3 16 | import sys 17 | 18 | from aiorpcx import TaskGroup 19 | from aiorpcx import ( 20 | JSONRPCAutoDetect, 21 | JSONRPCConnection, 22 | Request, 23 | RPCError, 24 | RPCSession, 25 | handler_invocation, 26 | serve_rs, 27 | NewlineFramer, 28 | ) 29 | 30 | from . import merkle 31 | from .merkle import sha256 32 | 33 | BAD_REQUEST = 1 34 | DAEMON_ERROR = 2 35 | 36 | MAX_CHUNK_SIZE = 2016 37 | 38 | (CACHE_DB,) = sys.argv[1:] 39 | 40 | 41 | class Env: 42 | max_recv = 10**7 43 | max_send = 10**7 44 | donation_address = None 45 | 46 | 47 | VERSION = "electrs/0.999999" # HACK: let Sparrow use batching 48 | 49 | LOG = logging.getLogger() 50 | 51 | 52 | class Manager: 53 | def __init__(self): 54 | self.db = sqlite3.connect(CACHE_DB) 55 | self.merkle = merkle.Merkle() 56 | self.sync_queue: asyncio.Queue = asyncio.Queue(50) 57 | self.sessions: set[ElectrumSession] = set() 58 | 59 | async def notify_sessions(self): 60 | for session in self.sessions: 61 | try: 62 | await session.notify() 63 | except Exception: 64 | LOG.exception("failed to notify session #%d", session.session_id) 65 | 66 | async def latest_header(self) -> dict: 67 | j = await self.chaininfo() 68 | height = j["blocks"] 69 | raw = await self.raw_header(height) 70 | return {"hex": raw.hex(), "height": height} 71 | 72 | async def get(self, path, f): 73 | async with aiohttp.ClientSession() as session: 74 | async with session.get(f"http://localhost:8332/rest/{path}") as response: 75 | response.raise_for_status() 76 | return await f(response) 77 | 78 | async def chaininfo(self): 79 | return await self.get("chaininfo.json", lambda r: r.json()) 80 | 81 | async def get_history(self, hashx: bytes): 82 | query = """ 83 | SELECT DISTINCT 84 | t.tx_id, 85 | t.block_height 86 | FROM 87 | transactions t, 88 | history h 89 | WHERE 90 | t.block_offset = h.block_offset AND 91 | t.block_height = h.block_height AND 92 | h.script_hash = ? 93 | ORDER BY 94 | t.block_height ASC, 95 | t.block_offset ASC 96 | """ 97 | return self.db.execute(query, [hashx]).fetchall() 98 | 99 | async def subscribe(self, hashX: bytes): 100 | event = asyncio.Event() 101 | await self.sync_queue.put((hashX, event.set)) 102 | await event.wait() 103 | 104 | async def _merkle_branch(self, height, tx_hashes, tx_pos): 105 | branch, _root = self.merkle.branch_and_root(tx_hashes, tx_pos) 106 | branch = [hash_to_hex_str(hash) for hash in branch] 107 | return branch 108 | 109 | async def merkle_branch_for_tx_hash(self, height, tx_hash): 110 | """Return a triple (branch, tx_pos).""" 111 | tx_hashes = await self.tx_hashes_at_blockheight(height) 112 | try: 113 | tx_pos = tx_hashes.index(tx_hash) 114 | except ValueError: 115 | raise RPCError( 116 | BAD_REQUEST, 117 | f"tx {hash_to_hex_str(tx_hash)} not in block at height {height:,d}", 118 | ) 119 | branch = await self._merkle_branch(height, tx_hashes, tx_pos) 120 | return branch, tx_pos 121 | 122 | async def tx_hashes_at_blockheight(self, height): 123 | """Returns a pair (tx_hashes). 124 | 125 | tx_hashes is an ordered list of binary hashes. Raises RPCError. 126 | """ 127 | 128 | h = await self.get(f"blockhashbyheight/{height}.hex", lambda r: r.text()) 129 | j = await self.get(f"block/notxdetails/{h}.json", lambda r: r.json()) 130 | tx_hashes = [hex_str_to_hash(h) for h in j["tx"]] 131 | return tx_hashes 132 | 133 | async def getrawtransaction(self, tx_hash, verbose): 134 | assert verbose is False 135 | rows = self.db.execute( 136 | "SELECT tx_bytes FROM transactions WHERE tx_id = ?", [tx_hash] 137 | ).fetchall() 138 | if len(rows) != 1: 139 | raise RPCError(BAD_REQUEST, f"{tx_hash.hex()} not found in DB") 140 | return rows[0][0] 141 | 142 | async def raw_header(self, height: int): 143 | raw, _ = await self.raw_headers(height, count=1) 144 | return raw 145 | 146 | async def raw_headers(self, height: int, count: int): 147 | chunks = [] 148 | while count > 0: 149 | h = await self.get(f"blockhashbyheight/{height}.hex", lambda r: r.text()) 150 | chunk_size = min(count, MAX_CHUNK_SIZE // 2) 151 | chunk = await self.get(f"headers/{chunk_size}/{h}.bin", lambda r: r.read()) 152 | assert len(chunk) % 80 == 0 153 | chunk_size = len(chunk) // 80 154 | assert chunk_size <= count 155 | chunks.append(chunk) 156 | height += chunk_size 157 | count -= chunk_size 158 | 159 | raw = b"".join(chunks) 160 | return raw, len(raw) // 80 161 | 162 | 163 | HASHX_LEN = 32 164 | 165 | 166 | hex_to_bytes = bytes.fromhex 167 | 168 | 169 | def hash_to_hex_str(x): 170 | """Convert a big-endian binary hash to displayed hex string. 171 | 172 | Display form of a binary hash is reversed and converted to hex. 173 | """ 174 | return bytes(reversed(x)).hex() 175 | 176 | 177 | def hex_str_to_hash(x: str) -> bytes: 178 | """Convert a displayed hex string to a binary hash.""" 179 | return bytes(reversed(hex_to_bytes(x))) 180 | 181 | 182 | def scripthash_to_hashX(scripthash: str): 183 | try: 184 | bin_hash = hex_str_to_hash(scripthash) 185 | if len(bin_hash) == 32: 186 | return bin_hash[:HASHX_LEN] 187 | except (ValueError, TypeError): 188 | pass 189 | raise RPCError(BAD_REQUEST, f"{scripthash} is not a valid script hash") 190 | 191 | 192 | def non_negative_integer(value): 193 | """Return param value it is or can be converted to a non-negative 194 | integer, otherwise raise an RPCError.""" 195 | try: 196 | value = int(value) 197 | if value >= 0: 198 | return value 199 | except (ValueError, TypeError): 200 | pass 201 | raise RPCError(BAD_REQUEST, f"{value} should be a non-negative integer") 202 | 203 | 204 | def assert_boolean(value): 205 | """Return param value it is boolean otherwise raise an RPCError.""" 206 | if value in (False, True): 207 | return value 208 | raise RPCError(BAD_REQUEST, f"{value} should be a boolean value") 209 | 210 | 211 | def assert_tx_hash(value): 212 | """Raise an RPCError if the value is not a valid hexadecimal transaction hash. 213 | 214 | If it is valid, return it as 32-byte binary hash. 215 | """ 216 | try: 217 | raw_hash = hex_str_to_hash(value) 218 | if len(raw_hash) == 32: 219 | return raw_hash 220 | except (ValueError, TypeError): 221 | pass 222 | raise RPCError(BAD_REQUEST, f"{value} should be a transaction hash") 223 | 224 | 225 | class SessionBase(RPCSession): 226 | """Base class of ElectrumX JSON sessions. 227 | 228 | Each session runs its tasks in asynchronous parallelism with other 229 | sessions. 230 | """ 231 | 232 | log_me = False 233 | 234 | session_counter = itertools.count() 235 | 236 | def __init__( 237 | self, 238 | transport, 239 | ): 240 | connection = JSONRPCConnection(JSONRPCAutoDetect) 241 | super().__init__(transport, connection=connection) 242 | self.txs_sent = 0 243 | self.session_id = None 244 | self.session_id = next(self.session_counter) 245 | self.logger = logging.getLogger() 246 | 247 | def default_framer(self): 248 | return NewlineFramer(max_size=self.env.max_recv) 249 | 250 | async def connection_lost(self): 251 | """Handle client disconnection.""" 252 | await super().connection_lost() 253 | msg = "" 254 | if self._incoming_concurrency.max_concurrent < self.initial_concurrent * 0.8: 255 | msg += " whilst throttled" 256 | if self.send_size >= 1_000_000: 257 | msg += f". Sent {self.send_size:,d} bytes in {self.send_count:,d} messages" 258 | if msg: 259 | msg = "disconnected" + msg 260 | self.logger.info(msg) 261 | 262 | async def handle_request(self, request): 263 | """Handle an incoming request. ElectrumX doesn't receive 264 | notifications from client sessions. 265 | """ 266 | if isinstance(request, Request): 267 | handler = self.request_handlers.get(request.method) 268 | else: 269 | handler = None 270 | 271 | coro = handler_invocation(handler, request)() 272 | return await coro 273 | 274 | 275 | def version_string(ptuple): 276 | """Convert a version tuple such as (1, 2) to "1.2". 277 | There is always at least one dot, so (1, ) becomes "1.0".""" 278 | while len(ptuple) < 2: 279 | ptuple += (0,) 280 | return ".".join(str(p) for p in ptuple) 281 | 282 | 283 | class ElectrumSession(SessionBase): 284 | """A TCP server that handles incoming Electrum connections.""" 285 | 286 | PROTOCOL_MIN = (1, 4) 287 | PROTOCOL_MAX = (1, 4, 3) 288 | 289 | def __init__(self, *args, mgr: Manager, **kwargs): 290 | super().__init__(*args, **kwargs) 291 | self.env = Env() 292 | self.session_mgr = mgr 293 | self.subscribe_headers: dict | None = None 294 | self.connection.max_response_size = self.env.max_send 295 | self.hashX_subs = {} 296 | self.sv_seen = False 297 | self.set_request_handlers() 298 | self.is_peer = False 299 | self.protocol_tuple = self.PROTOCOL_MIN 300 | self.session_mgr.sessions.add(self) 301 | 302 | async def connection_lost(self): 303 | """Handle client disconnection.""" 304 | await super().connection_lost() 305 | self.session_mgr.sessions.remove(self) 306 | 307 | @classmethod 308 | def protocol_min_max_strings(cls): 309 | return [version_string(ver) for ver in (cls.PROTOCOL_MIN, cls.PROTOCOL_MAX)] 310 | 311 | @classmethod 312 | def server_features(cls, env): 313 | """Return the server features dictionary.""" 314 | hosts_dict = {} 315 | for service in env.report_services: 316 | port_dict = hosts_dict.setdefault(str(service.host), {}) 317 | if service.protocol not in port_dict: 318 | port_dict[f"{service.protocol}_port"] = service.port 319 | 320 | min_str, max_str = cls.protocol_min_max_strings() 321 | return { 322 | "hosts": hosts_dict, 323 | "pruning": None, 324 | "server_version": VERSION, 325 | "protocol_min": min_str, 326 | "protocol_max": max_str, 327 | "hash_function": "sha256", 328 | "services": [], 329 | } 330 | 331 | async def server_features_async(self): 332 | return self.server_features(self.env) 333 | 334 | @classmethod 335 | def server_version_args(cls): 336 | return [VERSION, cls.protocol_min_max_strings()] 337 | 338 | def protocol_version_string(self): 339 | return version_string(self.protocol_tuple) 340 | 341 | def unsubscribe_hashX(self, hashX): 342 | return self.hashX_subs.pop(hashX, None) 343 | 344 | async def subscribe_headers_result(self) -> dict: 345 | return await self.session_mgr.latest_header() 346 | 347 | async def headers_subscribe(self): 348 | self.subscribe_headers = await self.subscribe_headers_result() 349 | return self.subscribe_headers 350 | 351 | async def add_peer(self, features): 352 | pass 353 | 354 | async def peers_subscribe(self): 355 | return [] 356 | 357 | async def address_status(self, hashX): 358 | db_history = await self.session_mgr.get_history(hashX) 359 | 360 | status = "".join( 361 | f"{hash_to_hex_str(tx_hash)}:{height:d}:" for tx_hash, height in db_history 362 | ) 363 | 364 | return sha256(status.encode()).hex() if status else None 365 | 366 | async def hashX_subscribe(self, hashX, alias): 367 | await self.session_mgr.subscribe(hashX) 368 | 369 | # Store the subscription only after address_status succeeds 370 | result = await self.address_status(hashX) 371 | self.hashX_subs[hashX] = result 372 | return result 373 | 374 | async def notify(self): 375 | if self.subscribe_headers is not None: 376 | new_result = await self.subscribe_headers_result() 377 | if self.subscribe_headers != new_result: 378 | self.subscribe_headers = new_result 379 | await self.send_notification( 380 | "blockchain.headers.subscribe", (new_result,) 381 | ) 382 | 383 | for hashX in list(self.hashX_subs): 384 | new_status = await self.address_status(hashX) 385 | status = self.hashX_subs[hashX] 386 | if status != new_status: 387 | self.hashX_subs[hashX] = new_status 388 | await self.send_notification( 389 | "blockchain.scripthash.subscribe", (new_status,) 390 | ) 391 | 392 | async def confirmed_history(self, hashX): 393 | history = await self.session_mgr.get_history(hashX) 394 | conf = [ 395 | {"tx_hash": hash_to_hex_str(tx_hash), "height": height} 396 | for tx_hash, height in history 397 | ] 398 | return conf 399 | 400 | async def scripthash_get_history(self, scripthash): 401 | hashX = scripthash_to_hashX(scripthash) 402 | return await self.confirmed_history(hashX) 403 | 404 | async def scripthash_subscribe(self, scripthash): 405 | hashX = scripthash_to_hashX(scripthash) 406 | return await self.hashX_subscribe(hashX, scripthash) 407 | 408 | async def scripthash_unsubscribe(self, scripthash): 409 | hashX = scripthash_to_hashX(scripthash) 410 | return self.unsubscribe_hashX(hashX) is not None 411 | 412 | async def block_header(self, height): 413 | height = non_negative_integer(height) 414 | raw_header_hex = (await self.session_mgr.raw_header(height)).hex() 415 | return raw_header_hex 416 | 417 | async def block_headers(self, start_height, count): 418 | start_height = non_negative_integer(start_height) 419 | count = non_negative_integer(count) 420 | 421 | max_size = MAX_CHUNK_SIZE 422 | count = min(count, max_size) 423 | headers, count = await self.session_mgr.raw_headers(start_height, count) 424 | result = {"hex": headers.hex(), "count": count, "max": max_size} 425 | return result 426 | 427 | async def donation_address(self): 428 | return self.env.donation_address 429 | 430 | async def banner(self): 431 | return "" 432 | 433 | async def relayfee(self): 434 | return 0 435 | 436 | async def estimatefee(self, number, mode=None): 437 | return 0 438 | 439 | async def ping(self): 440 | return None 441 | 442 | async def server_version(self, client_name="", protocol_version=None): 443 | return VERSION, self.protocol_version_string() 444 | 445 | async def transaction_get(self, tx_hash, verbose=False): 446 | tx_hash = assert_tx_hash(tx_hash) 447 | if verbose not in (True, False): 448 | raise RPCError(BAD_REQUEST, '"verbose" must be a boolean') 449 | 450 | raw = await self.session_mgr.getrawtransaction(tx_hash, verbose) 451 | return raw.hex() 452 | 453 | async def transaction_merkle(self, tx_hash, height): 454 | tx_hash = assert_tx_hash(tx_hash) 455 | height = non_negative_integer(height) 456 | 457 | branch, tx_pos = await self.session_mgr.merkle_branch_for_tx_hash( 458 | height, tx_hash 459 | ) 460 | return {"block_height": height, "merkle": branch, "pos": tx_pos} 461 | 462 | async def compact_fee_histogram(self): 463 | return [] 464 | 465 | def set_request_handlers(self): 466 | handlers = { 467 | "blockchain.block.header": self.block_header, 468 | "blockchain.block.headers": self.block_headers, 469 | "blockchain.estimatefee": self.estimatefee, 470 | "blockchain.headers.subscribe": self.headers_subscribe, 471 | "blockchain.relayfee": self.relayfee, 472 | "blockchain.scripthash.get_history": self.scripthash_get_history, 473 | "blockchain.scripthash.subscribe": self.scripthash_subscribe, 474 | "blockchain.transaction.get": self.transaction_get, 475 | "blockchain.transaction.get_merkle": self.transaction_merkle, 476 | "mempool.get_fee_histogram": self.compact_fee_histogram, 477 | "server.add_peer": self.add_peer, 478 | "server.banner": self.banner, 479 | "server.donation_address": self.donation_address, 480 | "server.features": self.server_features_async, 481 | "server.peers.subscribe": self.peers_subscribe, 482 | "server.ping": self.ping, 483 | "server.version": self.server_version, 484 | } 485 | handlers["blockchain.scripthash.unsubscribe"] = self.scripthash_unsubscribe 486 | 487 | self.request_handlers = handlers 488 | 489 | 490 | async def get_items(q: asyncio.Queue): 491 | items = [] 492 | timeout = 1.0 493 | try: 494 | while True: 495 | item = await asyncio.wait_for(q.get(), timeout) 496 | items.append(item) 497 | while not q.empty(): 498 | items.append(q.get_nowait()) 499 | # use a shorter timeout for coalescing subsequent subscriptions 500 | timeout = 0.01 501 | except asyncio.exceptions.TimeoutError: 502 | pass 503 | return items 504 | 505 | 506 | def update_scripthashes(c: sqlite3.Cursor, data: list[bytes]): 507 | # update bindex DB with the new scripthashes 508 | c.execute("BEGIN") 509 | try: 510 | r = c.executemany("INSERT OR IGNORE INTO watch (script_hash) VALUES (?1)", data) 511 | if r.rowcount: 512 | LOG.info("watching %d new addresses", r.rowcount) 513 | c.execute("COMMIT") 514 | except Exception: 515 | c.execute("ROLLBACK") 516 | raise 517 | 518 | 519 | class Indexer: 520 | def __init__(self): 521 | self.tip = None 522 | self._loop = asyncio.get_running_loop() 523 | 524 | async def _readline(self) -> str: 525 | return await self._loop.run_in_executor(None, sys.stdin.readline) 526 | 527 | async def _write(self, data: bytes): 528 | def write_fn(): 529 | sys.stdout.write(data) 530 | sys.stdout.flush() 531 | 532 | await self._loop.run_in_executor(None, write_fn) 533 | 534 | @classmethod 535 | async def start(cls) -> "Indexer": 536 | i = Indexer() 537 | line = await i._readline() # wait for an index sync 538 | i.tip = line.strip() 539 | LOG.info("indexer at block=%r", i.tip) 540 | return i 541 | 542 | async def sync(self) -> bool: 543 | # update history index for the new scripthashes 544 | await self._write("\n") 545 | 546 | # wait for the index sync to finish 547 | line = await self._readline() 548 | prev_tip = self.tip 549 | self.tip = line.strip() 550 | LOG.debug("indexer at block=%r", self.tip) 551 | return prev_tip != self.tip 552 | 553 | 554 | async def sync_task(mgr: Manager, indexer: Indexer): 555 | try: 556 | # sending new scripthashes on subscription requests 557 | while True: 558 | items = await get_items(mgr.sync_queue) 559 | data = [[i] for i, _ in items] 560 | update_scripthashes(mgr.db.cursor(), data) 561 | 562 | # update history index for the new scripthashesfinish 563 | chain_updated = await indexer.sync() 564 | if items or chain_updated: 565 | LOG.info("indexer at block=%r: %d reqs", indexer.tip, len(items)) 566 | 567 | # mark subscription requests as done 568 | for _, ack_fn in items: 569 | ack_fn() 570 | 571 | # make sure all sessions are notified 572 | if chain_updated: 573 | await mgr.notify_sessions() 574 | except Exception: 575 | LOG.exception("sync_task() failed") 576 | 577 | 578 | async def main(): 579 | FMT = "[%(asctime)-27s %(levelname)-5s %(module)s] %(message)s" 580 | logging.basicConfig(level="INFO", format=FMT) 581 | indexer = await Indexer.start() # wait for initial sync 582 | mgr = Manager() 583 | cls = functools.partial(ElectrumSession, mgr=mgr) 584 | await serve_rs(cls, host="localhost", port=50001) 585 | async with TaskGroup() as g: 586 | await g.spawn(sync_task(mgr, indexer)) 587 | await g.join() 588 | 589 | 590 | if __name__ == "__main__": 591 | asyncio.run(main()) 592 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd `dirname $0` 3 | set -eux 4 | export RUST_LOG=${RUST_LOG:-info} 5 | cargo +stable build --release --all --locked 6 | target/release/bindex-cli $* 7 | --------------------------------------------------------------------------------